diff --git a/.gitignore b/.gitignore index 5dd65a7..cbe5b45 100644 --- a/.gitignore +++ b/.gitignore @@ -76,4 +76,13 @@ matlab_runners/Head Pose Experiments/experiments/bu_out/ matlab_runners/Head Pose Experiments/experiments/ict_out/ OpenFace\.VC\.db matlab_version/face_validation/vlfeat-0.9.20/ -matlab_version/face_validation/trained/intermediate/ \ No newline at end of file +matlab_version/face_validation/trained/intermediate/ +lib/local/GazeAnalyser/x64/ +lib/local/Utilities/x64/ +exe/FeatureExtraction/processed/ +exe/FaceLandmarkImg/processed/ +exe/FeatureExtraction/processed_features/ +matlab_runners/Demos/processed_features/ +*.db +exe/releases/OpenFace_0.3.0_win_x64/ +exe/releases/OpenFace_0.3.0_win_x86/ diff --git a/.travis.yml b/.travis.yml index 9ae1e4d..8c9f66d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ language: cpp +dist: trusty +sudo: required + branches: only: - master @@ -9,32 +12,29 @@ compiler: - gcc os: - - osx - linux before_install: # OpenCV dependencies and boost - if [ ${TRAVIS_OS_NAME} = linux ]; then + sudo apt-get update; + sudo apt-get install libopenblas-dev; sudo apt-get install git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev; sudo apt-get install python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libdc1394-22-dev checkinstall; - sudo add-apt-repository -y ppa:boost-latest/ppa; - sudo apt-get update; + sudo apt-get install cmake; sudo apt-get install libboost1.55-all-dev; wget https://github.com/Itseez/opencv/archive/3.1.0.zip; - sudo unzip 3.1.0.zip; + unzip -q 3.1.0.zip; cd opencv-3.1.0; - sudo mkdir build; + mkdir build; cd build; fi # g++4.8.1 - if [ "$CXX" = "g++" ]; then if [ ${TRAVIS_OS_NAME} = linux ]; then - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; - sudo apt-get update -qq; - sudo apt-get install -qq g++-4.8; - export CXX="g++-4.8"; + $CXX --version; sudo cmake -D CMAKE_BUILD_TYPE=RELEASE -D WITH_V4L=ON -D WITH_OPENCL=OFF -D INSTALL_C_EXAMPLES=OFF -D BUILD_EXAMPLES=OFF -D BUILD_TESTS=OFF -D BUILD_PERF_TESTS=OFF -D BUILD_EXAMPLES=OFF -D INSTALL_PYTHON_EXAMPLES=OFF ..; fi fi @@ -43,16 +43,17 @@ before_install: - if [ "$CXX" = "clang++" ]; then if [ ${TRAVIS_OS_NAME} = linux ]; then $CXX --version; - sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers; sudo cmake -D CMAKE_BUILD_TYPE=RELEASE -D WITH_V4L=ON -D WITH_OPENCL=OFF -D INSTALL_C_EXAMPLES=OFF -D BUILD_EXAMPLES=OFF -D BUILD_TESTS=OFF -D BUILD_PERF_TESTS=OFF -D BUILD_EXAMPLES=OFF -D INSTALL_PYTHON_EXAMPLES=OFF -D WITH_TBB=ON ..; fi fi - - if [ ${TRAVIS_OS_NAME} = osx ]; then + - if [ ${TRAVIS_OS_NAME} = osx ]; then brew update; brew install tbb; + brew tap homebrew/science; + brew install vtk; wget https://github.com/Itseez/opencv/archive/3.1.0.zip; - sudo unzip 3.1.0.zip; + unzip -q 3.1.0.zip; cd opencv-3.1.0; sudo mkdir build; cd build; @@ -70,9 +71,7 @@ script: - cd build - cmake -D CMAKE_BUILD_TYPE=RELEASE CMAKE_CXX_FLAGS="-std=c++11" -D CMAKE_EXE_LINKER_FLAGS="-std=c++11" .. - make -j2 - - ../build/bin/FaceLandmarkImg -fdir "../samples/" -ofdir "./demo_img/" -oidir "./demo_img/" -wild -q - - ../build/bin/FaceLandmarkImg -inroot ../samples -f sample1.jpg -outroot data -of sample1.txt -op sample1.3d -oi sample1.bmp -multi_view 1 -wild -q - - ../build/bin/FaceLandmarkVidMulti -inroot ../samples -f multi_face.avi -outroot output -ov multi_face.avi -q - - ../build/bin/FeatureExtraction -f "../samples/2015-10-15-15-14.avi" -outroot output_features -ov default.avi -of "default.txt" -simalign aligned -ov feat_test.avi -hogalign hog_test.dat -q - - ../build/bin/FeatureExtraction -f "../samples/2015-10-15-15-14.avi" -outroot output_features -simsize 200 -simscale 0.5 -ov default.avi -of "default.txt" -simalign aligned -ov feat_test.avi -hogalign hog_test.dat -q - - ../build/bin/FaceLandmarkVid -inroot ../samples -f 2015-10-15-15-14.avi -f multi_face.avi -outroot output_data -ov 1.avi -ov 2.avi -q + - ../build/bin/FaceLandmarkImg -fdir "../samples/" -out_dir "./demo_img" -wild -q + - ../build/bin/FaceLandmarkImg -inroot ../samples -f sample1.jpg -out_dir data -of sample1.txt -multi_view 1 -wild -q + - ../build/bin/FeatureExtraction -f "../samples/2015-10-15-15-14.avi" -out_dir output -q + - ../build/bin/FeatureExtraction -f "../samples/2015-10-15-15-14.avi" -simsize 200 -simscale 0.5 -q diff --git a/CMakeLists.txt b/CMakeLists.txt index 6927d73..385e1c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ find_package( BLAS REQUIRED ) include_directories( ${BLAS_INCLUDE_DIRS} ) LINK_DIRECTORIES(${BLAS_LIBRARY_DIRS}) -find_package( OpenCV REQUIRED ) +find_package( OpenCV 3.1 REQUIRED ) MESSAGE("OpenCV information:") MESSAGE(" OpenCV_INCLUDE_DIRS: ${OpenCV_INCLUDE_DIRS}") @@ -191,6 +191,8 @@ add_subdirectory(lib/local/LandmarkDetector) add_subdirectory(lib/local/FaceAnalyser) # Gaze library add_subdirectory(lib/local/GazeAnalyser) +# Utilities library +add_subdirectory(lib/local/Utilities) # executables add_subdirectory(exe/FaceLandmarkImg) add_subdirectory(exe/FaceLandmarkVid) diff --git a/OpenFace.sln b/OpenFace.sln index 2ec8dc5..ffe01dd 100644 --- a/OpenFace.sln +++ b/OpenFace.sln @@ -39,6 +39,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenFaceDemo", "gui\OpenFac EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeadPoseLive", "gui\HeadPose-live\HeadPoseLive.csproj", "{F396362D-821E-4EA6-9BBF-1F6050844118}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Utilities", "lib\local\Utilities\Utilities.vcxproj", "{8E741EA2-9386-4CF2-815E-6F9B08991EAC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -158,6 +160,14 @@ Global {5F915541-F531-434F-9C81-79F5DB58012B}.Release|Win32.Build.0 = Release|Win32 {5F915541-F531-434F-9C81-79F5DB58012B}.Release|x64.ActiveCfg = Release|x64 {5F915541-F531-434F-9C81-79F5DB58012B}.Release|x64.Build.0 = Release|x64 + {8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|Win32.ActiveCfg = Debug|Win32 + {8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|Win32.Build.0 = Debug|Win32 + {8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|x64.ActiveCfg = Debug|x64 + {8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|x64.Build.0 = Debug|x64 + {8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|Win32.ActiveCfg = Release|Win32 + {8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|Win32.Build.0 = Release|Win32 + {8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|x64.ActiveCfg = Release|x64 + {8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -177,5 +187,6 @@ Global {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90} = {C9D8D0B0-11EC-41CB-8524-2DDB5BE94297} {F396362D-821E-4EA6-9BBF-1F6050844118} = {C9D8D0B0-11EC-41CB-8524-2DDB5BE94297} {5F915541-F531-434F-9C81-79F5DB58012B} = {99FEBA13-BDDF-4076-B57E-D8EF4076E20D} + {8E741EA2-9386-4CF2-815E-6F9B08991EAC} = {99FEBA13-BDDF-4076-B57E-D8EF4076E20D} EndGlobalSection EndGlobal diff --git a/README.md b/README.md index 5bbfdb8..02b9fd6 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,21 @@ -# OpenFace: an open source facial behavior analysis toolkit +# OpenFace 0.3.0: an open source facial behavior analysis toolkit [![Build Status](https://travis-ci.org/TadasBaltrusaitis/OpenFace.svg?branch=master)](https://travis-ci.org/TadasBaltrusaitis/OpenFace) [![Build status](https://ci.appveyor.com/api/projects/status/8msiklxfbhlnsmxp/branch/master?svg=true)](https://ci.appveyor.com/project/TadasBaltrusaitis/openface/branch/master) -Over the past few years, there has been an increased interest in automatic facial behavior analysis and understanding. We present OpenFace – an open source tool intended for computer vision and machine learning researchers, affective computing community and people interested in building interactive applications based on facial behavior analysis. OpenFace is the first open source tool capable of facial landmark detection, head pose estimation, facial action unit recognition, and eye-gaze estimation. The computer vision algorithms which represent the core of OpenFace demonstrate state-of-the-art results in all of the above mentioned tasks. Furthermore, our tool is capable of real-time performance and is able to run from a simple webcam without any specialist hardware. +Over the past few years, there has been an increased interest in automatic facial behavior analysis and understanding. We present OpenFace – a tool intended for computer vision and machine learning researchers, affective computing community and people interested in building interactive applications based on facial behavior analysis. OpenFace is the first toolkit capable of facial landmark detection, head pose estimation, facial action unit recognition, and eye-gaze estimation with available source code. The computer vision algorithms which represent the core of OpenFace demonstrate state-of-the-art results in all of the above mentioned tasks. Furthermore, our tool is capable of real-time performance and is able to run from a simple webcam without any specialist hardware. -The code was written mainly by Tadas Baltrusaitis during his time at the Language Technologies Institute at the Carnegie Mellon University; Computer Laboratory, University of Cambridge; and Institute for Creative Technologies, University of Southern California. +![Multicomp logo](https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/muticomp_logo_black.png) -Special thanks goes to Louis-Philippe Morency and his MultiComp Lab at Institute for Creative Technologies for help in writing and testing the code, and Erroll Wood for the gaze estimation work. +![Rainbow logo](https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/rainbow-logo.gif) + +OpenFace is an implementation of a number of research papers from the Multicomp group, Language Technologies Institute at the Carnegie Mellon University and Rainbow Group, Computer Laboratory, University of Cambridge. The founder of the project and main developer is Tadas Baltrušaitis. + +Special thanks goes to Louis-Philippe Morency and his MultiComp Lab at Carnegie Mellon University for help in writing and testing the code, and Erroll Wood for the gaze estimation work. ## WIKI **For instructions of how to install/compile/use the project please see [WIKI](https://github.com/TadasBaltrusaitis/OpenFace/wiki)** -More details about the project - http://www.cl.cam.ac.uk/research/rainbow/projects/openface/ - ## Functionality The system is capable of performing a number of facial analysis tasks: @@ -68,17 +70,18 @@ Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson in *Facial Expression Recognition and Analysis Challenge*, *IEEE International Conference on Automatic Face and Gesture Recognition*, 2015 -# Copyright - -Copyright can be found in the Copyright.txt - -You have to respect boost, TBB, dlib, OpenBLAS, and OpenCV licenses. - # Commercial license For inquiries about the commercial licensing of the OpenFace toolkit please contact innovation@cmu.edu # Final remarks -I did my best to make sure that the code runs out of the box but there are always issues and I would be grateful for your understanding that this is research code and not full fledged product. However, if you encounter any problems/bugs/issues please contact me on github or by emailing me at Tadas.Baltrusaitis@cl.cam.ac.uk for any bug reports/questions/suggestions. +I did my best to make sure that the code runs out of the box but there are always issues and I would be grateful for your understanding that this is research code and not full fledged product. However, if you encounter any problems/bugs/issues please contact me on github or by emailing me at Tadas.Baltrusaitis@cl.cam.ac.uk for any bug reports/questions/suggestions. I prefer questions and bug reports on github as that provides visibility to others who might be encountering same issues or who have the same questions. +# Copyright + +Copyright can be found in the Copyright.txt + +You have to respect boost, TBB, dlib, OpenBLAS, and OpenCV licenses. + +Furthermore you have to respect the licenses of the datasets used for model training - https://github.com/TadasBaltrusaitis/OpenFace/wiki/Datasets diff --git a/appveyor.yml b/appveyor.yml index eb5d882..d59ae35 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,7 +6,6 @@ branches: - /^feature-.*$/ max_jobs: 4 configuration: -- Debug - Release platform: - x64 @@ -21,9 +20,9 @@ test_script: - cmd: if exist Debug (cd Debug) - cmd: if exist Release (cd Release) - cmd: dir - - cmd: if exist "../samples" (FaceLandmarkImg.exe -fdir "../samples/" -ofdir "./demo_img/" -oidir "./demo_img/" -wild -q) else (FaceLandmarkImg.exe -fdir "../../samples/" -ofdir "./demo_img/" -oidir "./demo_img/" -wild -q) - - cmd: if exist "../samples" (FaceLandmarkImg.exe -inroot ../samples -f sample1.jpg -outroot out_data -of sample1.pts -op sample1.3d -oi sample1.bmp -q) else (FaceLandmarkImg.exe -inroot ../../samples -f sample1.jpg -outroot out_data -of sample1.pts -op sample1.3d -oi sample1.bmp -q) - - cmd: if exist "../samples" (FaceLandmarkVidMulti.exe -inroot ../samples -f multi_face.avi -ov multi_face.avi -q) else (FaceLandmarkVidMulti.exe -inroot ../../samples -f multi_face.avi -ov multi_face.avi -q) - - cmd: if exist "../samples" (FeatureExtraction.exe -f "../samples/default.wmv" -outroot output_features -of "default.txt" -simalign aligned -ov feat_track.avi -hogalign hog_test.dat -q) else (FeatureExtraction.exe -f "../../samples/default.wmv" -outroot output_features -of "default.txt" -simalign aligned -ov feat_track.avi -hogalign hog_test.dat -q) - - cmd: if exist "../samples" (FeatureExtraction.exe -f "../samples/default.wmv" -outroot output_features -of "default.txt" -simalign aligned -simsize 200 -simscale 0.5 -ov feat_track.avi -hogalign hog_test.dat -q) else (FeatureExtraction.exe -f "../../samples/default.wmv" -outroot output_features -of "default.txt" -simalign aligned -simsize 200 -simscale 0.5 -ov feat_track.avi -hogalign hog_test.dat -q) - - cmd: if exist "../samples" (FaceLandmarkVid.exe -f "../samples/default.wmv" -ov track.avi -q) else (FaceLandmarkVid.exe -f "../../samples/default.wmv" -ov track.avi -q) + - cmd: if exist "../samples" (FaceLandmarkImg.exe -fdir "../samples/" -out_dir "./demo_img/" -wild -q) else (FaceLandmarkImg.exe -fdir "../../samples/" -out_dir "./demo_img/" -wild -q) + - cmd: if exist "../samples" (FaceLandmarkImg.exe -inroot ../samples -f sample1.jpg -out_dir out_data -of sample1.pts -q) else (FaceLandmarkImg.exe -inroot ../../samples -f sample1.jpg -out_dir out_data -q) + - cmd: if exist "../samples" (FaceLandmarkVidMulti.exe -inroot ../samples -f multi_face.avi -q) else (FaceLandmarkVidMulti.exe -inroot ../../samples -f multi_face.avi -q) + - cmd: if exist "../samples" (FeatureExtraction.exe -f "../samples/default.wmv" -q) else (FeatureExtraction.exe -f "../../samples/default.wmv" -q) + - cmd: if exist "../samples" (FeatureExtraction.exe -f "../samples/default.wmv" -simsize 200 -simscale 0.5 -q) else (FeatureExtraction.exe -f "../../samples/default.wmv" -simsize 200 -simscale 0.5 -ov feat_track.avi -q) + - cmd: if exist "../samples" (FaceLandmarkVid.exe -f "../samples/default.wmv" -q) else (FaceLandmarkVid.exe -f "../../samples/default.wmv" -q) diff --git a/exe/FaceLandmarkImg/CMakeLists.txt b/exe/FaceLandmarkImg/CMakeLists.txt index a2af8df..e986ef4 100644 --- a/exe/FaceLandmarkImg/CMakeLists.txt +++ b/exe/FaceLandmarkImg/CMakeLists.txt @@ -7,11 +7,13 @@ include_directories(${LandmarkDetector_SOURCE_DIR}/include) include_directories(../../lib/local/LandmarkDetector/include) include_directories(../../lib/local/FaceAnalyser/include) include_directories(../../lib/local/GazeAnalyser/include) +include_directories(../../lib/local/Utilities/include) add_executable(FaceLandmarkImg FaceLandmarkImg.cpp) target_link_libraries(FaceLandmarkImg LandmarkDetector) target_link_libraries(FaceLandmarkImg FaceAnalyser) target_link_libraries(FaceLandmarkImg GazeAnalyser) +target_link_libraries(FaceLandmarkImg Utilities) target_link_libraries(FaceLandmarkImg dlib) target_link_libraries(FaceLandmarkImg ${OpenCV_LIBS} ${Boost_LIBRARIES} ${TBB_LIBRARIES} ${BLAS_LIBRARIES}) diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index 8050611..3980c93 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -54,6 +54,13 @@ #include #include +#include +#include +#include +#include +#include + + #ifndef CONFIG_DIR #define CONFIG_DIR "~" #endif @@ -72,337 +79,72 @@ vector get_arguments(int argc, char **argv) return arguments; } -void convert_to_grayscale(const cv::Mat& in, cv::Mat& out) -{ - if(in.channels() == 3) - { - // Make sure it's in a correct format - if(in.depth() != CV_8U) - { - if(in.depth() == CV_16U) - { - cv::Mat tmp = in / 256; - tmp.convertTo(tmp, CV_8U); - cv::cvtColor(tmp, out, CV_BGR2GRAY); - } - } - else - { - cv::cvtColor(in, out, CV_BGR2GRAY); - } - } - else if(in.channels() == 4) - { - cv::cvtColor(in, out, CV_BGRA2GRAY); - } - else - { - if(in.depth() == CV_16U) - { - cv::Mat tmp = in / 256; - out = tmp.clone(); - } - else if(in.depth() != CV_8U) - { - in.convertTo(out, CV_8U); - } - else - { - out = in.clone(); - } - } -} - -// Useful utility for creating directories for storing the output files -void create_directory_from_file(string output_path) -{ - - // Creating the right directory structure - - // First get rid of the file - auto p = boost::filesystem::path(boost::filesystem::path(output_path).parent_path()); - - if (!p.empty() && !boost::filesystem::exists(p)) - { - bool success = boost::filesystem::create_directories(p); - if (!success) - { - cout << "Failed to create a directory... " << p.string() << endl; - } - } -} - -// This will only be accurate when camera parameters are accurate, useful for work on 3D data -void write_out_pose_landmarks(const string& outfeatures, const cv::Mat_& shape3D, const cv::Vec6d& pose, const cv::Point3f& gaze0, const cv::Point3f& gaze1) -{ - create_directory_from_file(outfeatures); - std::ofstream featuresFile; - featuresFile.open(outfeatures); - - if (featuresFile.is_open()) - { - int n = shape3D.cols; - featuresFile << "version: 1" << endl; - featuresFile << "npoints: " << n << endl; - featuresFile << "{" << endl; - - for (int i = 0; i < n; ++i) - { - // Use matlab format, so + 1 - featuresFile << shape3D.at(i) << " " << shape3D.at(i + n) << " " << shape3D.at(i + 2*n) << endl; - } - featuresFile << "}" << endl; - - // Do the pose and eye gaze if present as well - featuresFile << "pose: eul_x, eul_y, eul_z: " << endl; - featuresFile << "{" << endl; - featuresFile << pose[3] << " " << pose[4] << " " << pose[5] << endl; - featuresFile << "}" << endl; - - // Do the pose and eye gaze if present as well - featuresFile << "gaze_vec: dir_x_1, dir_y_1, dir_z_1, dir_x_2, dir_y_2, dir_z_2: " << endl; - featuresFile << "{" << endl; - featuresFile << gaze0.x << " " << gaze0.y << " " << gaze0.z << " " << gaze1.x << " " << gaze1.y << " " << gaze1.z << endl; - featuresFile << "}" << endl; - - featuresFile.close(); - } -} - -void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF& clnf_model, const cv::Vec6d& pose, const cv::Point3f& gaze0, const cv::Point3f& gaze1, const cv::Vec2d gaze_angle, std::vector> au_intensities, std::vector> au_occurences, bool output_gaze) -{ - create_directory_from_file(outfeatures); - std::ofstream featuresFile; - featuresFile.open(outfeatures); - - if (featuresFile.is_open()) - { - int n = clnf_model.patch_experts.visibilities[0][0].rows; - featuresFile << "version: 2" << endl; - featuresFile << "npoints: " << n << endl; - featuresFile << "{" << endl; - - for (int i = 0; i < n; ++i) - { - // Use matlab format, so + 1 - featuresFile << clnf_model.detected_landmarks.at(i) + 1 << " " << clnf_model.detected_landmarks.at(i + n) + 1 << endl; - } - featuresFile << "}" << endl; - - // Do the pose and eye gaze if present as well - featuresFile << "pose: eul_x, eul_y, eul_z: " << endl; - featuresFile << "{" << endl; - featuresFile << pose[3] << " " << pose[4] << " " << pose[5] << endl; - featuresFile << "}" << endl; - - if(output_gaze) - { - featuresFile << "gaze: dir_x_1, dir_y_1, dir_z_1, dir_x_2, dir_y_2, dir_z_2: " << endl; - featuresFile << "{" << endl; - featuresFile << gaze0.x << " " << gaze0.y << " " << gaze0.z << " " << gaze1.x << " " << gaze1.y << " " << gaze1.z << endl; - featuresFile << "}" << endl; - - featuresFile << "gaze: angle_x, angle_y: " << endl; - featuresFile << "{" << endl; - featuresFile << gaze_angle[0] << " " << gaze_angle[1] << endl; - featuresFile << "}" << endl; - - std::vector eye_landmark_points = LandmarkDetector::CalculateAllEyeLandmarks(clnf_model); - - featuresFile << "eye_lmks: " << eye_landmark_points.size() << endl; - featuresFile << "{" << endl; - - for (int i = 0; i < eye_landmark_points.size(); ++i) - { - // Use matlab format, so + 1 - featuresFile << (eye_landmark_points[i].x + 1) << " " << (eye_landmark_points[i].y + 1) << endl; - } - featuresFile << "}" << endl; - } - // Do the au intensities - featuresFile << "au intensities: " << au_intensities.size() << endl; - featuresFile << "{" << endl; - - for (size_t i = 0; i < au_intensities.size(); ++i) - { - // Use matlab format, so + 1 - featuresFile << au_intensities[i].first << " " << au_intensities[i].second << endl; - } - - featuresFile << "}" << endl; - - // Do the au occurences - featuresFile << "au occurences: " << au_occurences.size() << endl; - featuresFile << "{" << endl; - - for (size_t i = 0; i < au_occurences.size(); ++i) - { - // Use matlab format, so + 1 - featuresFile << au_occurences[i].first << " " << au_occurences[i].second << endl; - } - - featuresFile << "}" << endl; - - - featuresFile.close(); - } -} - -void create_display_image(const cv::Mat& orig, cv::Mat& display_image, LandmarkDetector::CLNF& clnf_model) -{ - - // Draw head pose if present and draw eye gaze as well - - // preparing the visualisation image - display_image = orig.clone(); - - // Creating a display image - cv::Mat xs = clnf_model.detected_landmarks(cv::Rect(0, 0, 1, clnf_model.detected_landmarks.rows/2)); - cv::Mat ys = clnf_model.detected_landmarks(cv::Rect(0, clnf_model.detected_landmarks.rows/2, 1, clnf_model.detected_landmarks.rows/2)); - double min_x, max_x, min_y, max_y; - - cv::minMaxLoc(xs, &min_x, &max_x); - cv::minMaxLoc(ys, &min_y, &max_y); - - double width = max_x - min_x; - double height = max_y - min_y; - - int minCropX = max((int)(min_x-width/3.0),0); - int minCropY = max((int)(min_y-height/3.0),0); - - int widthCrop = min((int)(width*5.0/3.0), display_image.cols - minCropX - 1); - int heightCrop = min((int)(height*5.0/3.0), display_image.rows - minCropY - 1); - - double scaling = 350.0/widthCrop; - - // first crop the image - display_image = display_image(cv::Rect((int)(minCropX), (int)(minCropY), (int)(widthCrop), (int)(heightCrop))); - - // now scale it - cv::resize(display_image.clone(), display_image, cv::Size(), scaling, scaling); - - // Make the adjustments to points - xs = (xs - minCropX)*scaling; - ys = (ys - minCropY)*scaling; - - cv::Mat shape = clnf_model.detected_landmarks.clone(); - - xs.copyTo(shape(cv::Rect(0, 0, 1, clnf_model.detected_landmarks.rows/2))); - ys.copyTo(shape(cv::Rect(0, clnf_model.detected_landmarks.rows/2, 1, clnf_model.detected_landmarks.rows/2))); - - // Do the shifting for the hierarchical models as well - for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part) - { - cv::Mat xs = clnf_model.hierarchical_models[part].detected_landmarks(cv::Rect(0, 0, 1, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2)); - cv::Mat ys = clnf_model.hierarchical_models[part].detected_landmarks(cv::Rect(0, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2, 1, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2)); - - xs = (xs - minCropX)*scaling; - ys = (ys - minCropY)*scaling; - - cv::Mat shape = clnf_model.hierarchical_models[part].detected_landmarks.clone(); - - xs.copyTo(shape(cv::Rect(0, 0, 1, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2))); - ys.copyTo(shape(cv::Rect(0, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2, 1, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2))); - - } - - LandmarkDetector::Draw(display_image, clnf_model); - -} - int main (int argc, char **argv) { //Convert arguments to more convenient vector form vector arguments = get_arguments(argc, argv); - // Some initial parameters that can be overriden from command line - vector files, output_images, output_landmark_locations, output_pose_locations; + // Prepare for image reading + Utilities::ImageCapture image_reader; - // Bounding boxes for a face in each image (optional) - vector > bounding_boxes; - - LandmarkDetector::get_image_input_output_params(files, output_landmark_locations, output_pose_locations, output_images, bounding_boxes, arguments); - LandmarkDetector::FaceModelParameters det_parameters(arguments); - // No need to validate detections, as we're not doing tracking - det_parameters.validate_detections = false; - - // Grab camera parameters if provided (only used for pose and eye gaze and are quite important for accurate estimates) - float fx = 0, fy = 0, cx = 0, cy = 0; - int device = -1; - LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments); - - // If cx (optical axis centre) is undefined will use the image size/2 as an estimate - bool cx_undefined = false; - bool fx_undefined = false; - if (cx == 0 || cy == 0) + // The sequence reader chooses what to open based on command line arguments provided + if (!image_reader.Open(arguments)) { - cx_undefined = true; - } - if (fx == 0 || fy == 0) - { - fx_undefined = true; + cout << "Could not open any images" << endl; + return 1; } + // Load the models if images found + LandmarkDetector::FaceModelParameters det_parameters(arguments); + // The modules that are being used for tracking cout << "Loading the model" << endl; - LandmarkDetector::CLNF clnf_model(det_parameters.model_location); + LandmarkDetector::CLNF face_model(det_parameters.model_location); cout << "Model loaded" << endl; - - cv::CascadeClassifier classifier(det_parameters.face_detector_location); - dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector(); // Load facial feature extractor and AU analyser (make sure it is static) FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); face_analysis_params.OptimizeForImages(); FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); - bool visualise = !det_parameters.quiet_mode; + // If bounding boxes not provided, use a face detector + cv::CascadeClassifier classifier(det_parameters.face_detector_location); + dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector(); - // Do some image loading - for(size_t i = 0; i < files.size(); i++) + // A utility for visualizing the results + Utilities::Visualizer visualizer(arguments); + + cv::Mat captured_image; + + captured_image = image_reader.GetNextImage(); + + cout << "Starting tracking" << endl; + while (!captured_image.empty()) { - string file = files.at(i); - // Loading image - cv::Mat read_image = cv::imread(file, -1); + Utilities::RecorderOpenFaceParameters recording_params(arguments, false, false, + image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy); + Utilities::RecorderOpenFace open_face_rec(image_reader.name, recording_params, arguments); + + visualizer.SetImage(captured_image, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy); + + if (recording_params.outputGaze() && !face_model.eye_model) + cout << "WARNING: no eye model defined, but outputting gaze" << endl; - if (read_image.empty()) - { - cout << "Could not read the input image" << endl; - return 1; - } - // Making sure the image is in uchar grayscale - cv::Mat_ grayscale_image; - convert_to_grayscale(read_image, grayscale_image); - + cv::Mat_ grayscale_image = image_reader.GetGrayFrame(); - // If optical centers are not defined just use center of image - if (cx_undefined) + // Detect faces in an image + vector > face_detections; + + if (image_reader.has_bounding_boxes) { - cx = grayscale_image.cols / 2.0f; - cy = grayscale_image.rows / 2.0f; + face_detections = image_reader.GetBoundingBoxes(); } - // Use a rough guess-timate of focal length - if (fx_undefined) + else { - fx = 500 * (grayscale_image.cols / 640.0); - fy = 500 * (grayscale_image.rows / 480.0); - - fx = (fx + fy) / 2.0; - fy = fx; - } - - - // if no pose defined we just use a face detector - if(bounding_boxes.empty()) - { - - // Detect faces in an image - vector > face_detections; - - if(det_parameters.curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) + if (det_parameters.curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) { vector confidences; LandmarkDetector::DetectFacesHOG(face_detections, grayscale_image, face_detector_hog, confidences); @@ -411,199 +153,68 @@ int main (int argc, char **argv) { LandmarkDetector::DetectFaces(face_detections, grayscale_image, classifier); } - - // Detect landmarks around detected faces - int face_det = 0; - // perform landmark detection for every face detected - for(size_t face=0; face < face_detections.size(); ++face) - { - // if there are multiple detections go through them - bool success = LandmarkDetector::DetectLandmarksInImage(grayscale_image, face_detections[face], clnf_model, det_parameters); - - // Estimate head pose and eye gaze - cv::Vec6d headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); - - // Gaze tracking, absolute gaze direction - cv::Point3f gazeDirection0(0, 0, -1); - cv::Point3f gazeDirection1(0, 0, -1); - cv::Vec2d gazeAngle(0, 0); - - if (success && det_parameters.track_gaze) - { - GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); - GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); - gazeAngle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); - } - - auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model.detected_landmarks, false); - - // Writing out the detected landmarks (in an OS independent manner) - if(!output_landmark_locations.empty()) - { - char name[100]; - // append detection number (in case multiple faces are detected) - sprintf(name, "_det_%d", face_det); - - // Construct the output filename - boost::filesystem::path slash("/"); - std::string preferredSlash = slash.make_preferred().string(); - - boost::filesystem::path out_feat_path(output_landmark_locations.at(i)); - boost::filesystem::path dir = out_feat_path.parent_path(); - boost::filesystem::path fname = out_feat_path.filename().replace_extension(""); - boost::filesystem::path ext = out_feat_path.extension(); - string outfeatures = dir.string() + preferredSlash + fname.string() + string(name) + ext.string(); - write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, gazeAngle, ActionUnits.first, ActionUnits.second, det_parameters.track_gaze); - } - - if (!output_pose_locations.empty()) - { - char name[100]; - // append detection number (in case multiple faces are detected) - sprintf(name, "_det_%d", face_det); - - // Construct the output filename - boost::filesystem::path slash("/"); - std::string preferredSlash = slash.make_preferred().string(); - - boost::filesystem::path out_pose_path(output_pose_locations.at(i)); - boost::filesystem::path dir = out_pose_path.parent_path(); - boost::filesystem::path fname = out_pose_path.filename().replace_extension(""); - boost::filesystem::path ext = out_pose_path.extension(); - string outfeatures = dir.string() + preferredSlash + fname.string() + string(name) + ext.string(); - write_out_pose_landmarks(outfeatures, clnf_model.GetShape(fx, fy, cx, cy), headPose, gazeDirection0, gazeDirection1); - - } - - if (det_parameters.track_gaze) - { - cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); - - // Draw it in reddish if uncertain, blueish if certain - LandmarkDetector::DrawBox(read_image, pose_estimate_to_draw, cv::Scalar(255.0, 0, 0), 3, fx, fy, cx, cy); - GazeAnalysis::DrawGaze(read_image, clnf_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); - } - - // displaying detected landmarks - cv::Mat display_image; - create_display_image(read_image, display_image, clnf_model); - - if(visualise && success) - { - imshow("colour", display_image); - cv::waitKey(1); - } - - // Saving the display images (in an OS independent manner) - if(!output_images.empty() && success) - { - string outimage = output_images.at(i); - if(!outimage.empty()) - { - char name[100]; - sprintf(name, "_det_%d", face_det); - - boost::filesystem::path slash("/"); - std::string preferredSlash = slash.make_preferred().string(); - - // append detection number - boost::filesystem::path out_feat_path(outimage); - boost::filesystem::path dir = out_feat_path.parent_path(); - boost::filesystem::path fname = out_feat_path.filename().replace_extension(""); - boost::filesystem::path ext = out_feat_path.extension(); - outimage = dir.string() + preferredSlash + fname.string() + string(name) + ext.string(); - create_directory_from_file(outimage); - bool write_success = cv::imwrite(outimage, display_image); - - if (!write_success) - { - cout << "Could not output a processed image" << endl; - return 1; - } - - } - - } - - if(success) - { - face_det++; - } - - } } - else + + // Detect landmarks around detected faces + int face_det = 0; + // perform landmark detection for every face detected + for (size_t face = 0; face < face_detections.size(); ++face) { - // Have provided bounding boxes - LandmarkDetector::DetectLandmarksInImage(grayscale_image, bounding_boxes[i], clnf_model, det_parameters); + // if there are multiple detections go through them + bool success = LandmarkDetector::DetectLandmarksInImage(grayscale_image, face_detections[face], face_model, det_parameters); // Estimate head pose and eye gaze - cv::Vec6d headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy); // Gaze tracking, absolute gaze direction - cv::Point3f gazeDirection0(0, 0, -1); - cv::Point3f gazeDirection1(0, 0, -1); - cv::Vec2d gazeAngle(0, 0); + cv::Point3f gaze_direction0(0, 0, -1); + cv::Point3f gaze_direction1(0, 0, -1); + cv::Vec2d gaze_angle(0, 0); - if (det_parameters.track_gaze) + if (face_model.eye_model) { - GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); - GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); - gazeAngle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); + GazeAnalysis::EstimateGaze(face_model, gaze_direction0, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy, true); + GazeAnalysis::EstimateGaze(face_model, gaze_direction1, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy, false); + gaze_angle = GazeAnalysis::GetGazeAngle(gaze_direction0, gaze_direction1); } - auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model.detected_landmarks, false); + cv::Mat sim_warped_img; + cv::Mat_ hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0; - // Writing out the detected landmarks - if(!output_landmark_locations.empty()) + // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization + if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog) { - string outfeatures = output_landmark_locations.at(i); - write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, gazeAngle, ActionUnits.first, ActionUnits.second, det_parameters.track_gaze); + face_analyser.PredictStaticAUsAndComputeFeatures(captured_image, face_model.detected_landmarks); + face_analyser.GetLatestAlignedFace(sim_warped_img); + face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); } - // Writing out the detected landmarks - if (!output_pose_locations.empty()) - { - string outfeatures = output_pose_locations.at(i); - write_out_pose_landmarks(outfeatures, clnf_model.GetShape(fx, fy, cx, cy), headPose, gazeDirection0, gazeDirection1); - } + // Displaying the tracking visualizations + visualizer.SetObservationFaceAlign(sim_warped_img); + visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); + visualizer.SetObservationLandmarks(face_model.detected_landmarks, 1.0, face_model.detection_success); // Set confidence to high to make sure we always visualize + visualizer.SetObservationPose(pose_estimate, 1.0); + visualizer.SetObservationGaze(gaze_direction0, gaze_direction1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy), face_model.detection_certainty); - // displaying detected stuff - cv::Mat display_image; + // Setting up the recorder output + open_face_rec.SetObservationHOG(face_model.detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG + open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); + open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); + open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy), + face_model.params_global, face_model.params_local, face_model.detection_certainty, face_model.detection_success); + open_face_rec.SetObservationPose(pose_estimate); + open_face_rec.SetObservationGaze(gaze_direction0, gaze_direction1, gaze_angle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy)); + open_face_rec.SetObservationFaceAlign(sim_warped_img); + open_face_rec.WriteObservation(); - if (det_parameters.track_gaze) - { - cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); + } + if(face_detections.size() > 0) + { + visualizer.ShowObservation(); + } - // Draw it in reddish if uncertain, blueish if certain - LandmarkDetector::DrawBox(read_image, pose_estimate_to_draw, cv::Scalar(255.0, 0, 0), 3, fx, fy, cx, cy); - GazeAnalysis::DrawGaze(read_image, clnf_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); - } - - create_display_image(read_image, display_image, clnf_model); - - if(visualise) - { - imshow("colour", display_image); - cv::waitKey(1); - } - - if(!output_images.empty()) - { - string outimage = output_images.at(i); - if(!outimage.empty()) - { - create_directory_from_file(outimage); - bool write_success = imwrite(outimage, display_image); - - if (!write_success) - { - cout << "Could not output a processed image" << endl; - return 1; - } - } - } - } + // Grabbing the next frame in the sequence + captured_image = image_reader.GetNextImage(); } diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj b/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj index 08de70a..38bc8fd 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj @@ -112,7 +112,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 true @@ -127,7 +127,7 @@ Level3 Disabled WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions true @@ -144,7 +144,7 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false StreamingSIMDExtensions2 true @@ -162,12 +162,14 @@ NotUsing Full true - true + false WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false AdvancedVectorExtensions true + AnySuitable + Speed Console @@ -189,6 +191,9 @@ {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} + + {8e741ea2-9386-4cf2-815e-6f9b08991eac} + diff --git a/exe/FaceLandmarkVid/CMakeLists.txt b/exe/FaceLandmarkVid/CMakeLists.txt index ad822c3..eece095 100644 --- a/exe/FaceLandmarkVid/CMakeLists.txt +++ b/exe/FaceLandmarkVid/CMakeLists.txt @@ -9,10 +9,12 @@ include_directories(${LandmarkDetector_SOURCE_DIR}/include) include_directories(../../lib/local/LandmarkDetector/include) include_directories(../../lib/local/FaceAnalyser/include) include_directories(../../lib/local/GazeAnalyser/include) +include_directories(../../lib/local/Utilities/include) target_link_libraries(FaceLandmarkVid LandmarkDetector) target_link_libraries(FaceLandmarkVid FaceAnalyser) target_link_libraries(FaceLandmarkVid GazeAnalyser) +target_link_libraries(FaceLandmarkVid Utilities) target_link_libraries(FaceLandmarkVid dlib) diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index 8bda263..f394027 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -46,6 +46,10 @@ #include #include +#include +#include +#include + // Boost includes #include #include @@ -82,276 +86,113 @@ vector get_arguments(int argc, char **argv) return arguments; } -// Some globals for tracking timing information for visualisation -double fps_tracker = -1.0; -int64 t0 = 0; - -// Visualising the results -void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, int frame_count, double fx, double fy, double cx, double cy) -{ - - // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised - double detection_certainty = face_model.detection_certainty; - bool detection_success = face_model.detection_success; - - double visualisation_boundary = 0.2; - - // Only draw if the reliability is reasonable, the value is slightly ad-hoc - if (detection_certainty < visualisation_boundary) - { - LandmarkDetector::Draw(captured_image, face_model); - - double vis_certainty = detection_certainty; - if (vis_certainty > 1) - vis_certainty = 1; - if (vis_certainty < -1) - vis_certainty = -1; - - vis_certainty = (vis_certainty + 1) / (visualisation_boundary + 1); - - // A rough heuristic for box around the face width - int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); - - cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy); - - // Draw it in reddish if uncertain, blueish if certain - LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar((1 - vis_certainty)*255.0, 0, vis_certainty * 255), thickness, fx, fy, cx, cy); - - if (det_parameters.track_gaze && detection_success && face_model.eye_model) - { - GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); - } - } - - // Work out the framerate - if (frame_count % 10 == 0) - { - double t1 = cv::getTickCount(); - fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency()); - t0 = t1; - } - - // Write out the framerate on the image before displaying it - char fpsC[255]; - std::sprintf(fpsC, "%d", (int)fps_tracker); - string fpsSt("FPS:"); - fpsSt += fpsC; - cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0)); - - if (!det_parameters.quiet_mode) - { - cv::namedWindow("tracking_result", 1); - cv::imshow("tracking_result", captured_image); - } -} - int main (int argc, char **argv) { vector arguments = get_arguments(argc, argv); - // Some initial parameters that can be overriden from command line - vector files, output_video_files, out_dummy; - - // By default try webcam 0 - int device = 0; - LandmarkDetector::FaceModelParameters det_parameters(arguments); - // Get the input output file parameters - - // Indicates that rotation should be with respect to world or camera coordinates - string output_codec; - LandmarkDetector::get_video_input_output_params(files, out_dummy, output_video_files, output_codec, arguments); - // The modules that are being used for tracking - LandmarkDetector::CLNF clnf_model(det_parameters.model_location); + LandmarkDetector::CLNF face_model(det_parameters.model_location); - // Grab camera parameters, if they are not defined (approximate values will be used) - float fx = 0, fy = 0, cx = 0, cy = 0; - // Get camera parameters - LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments); + // Open a sequence + Utilities::SequenceCapture sequence_reader; - // If cx (optical axis centre) is undefined will use the image size/2 as an estimate - bool cx_undefined = false; - bool fx_undefined = false; - if (cx == 0 || cy == 0) + // A utility for visualizing the results (show just the tracks) + Utilities::Visualizer visualizer(true, false, false); + + // Tracking FPS for visualization + Utilities::FpsTracker fps_tracker; + fps_tracker.AddFrame(); + + int sequence_number = 0; + + while (true) // this is not a for loop as we might also be reading from a webcam { - cx_undefined = true; - } - if (fx == 0 || fy == 0) - { - fx_undefined = true; - } - // If multiple video files are tracked, use this to indicate if we are done - bool done = false; - int f_n = -1; - - det_parameters.track_gaze = true; - - while(!done) // this is not a for loop as we might also be reading from a webcam - { - - string current_file; - - // We might specify multiple video files as arguments - if(files.size() > 0) + // The sequence reader chooses what to open based on command line arguments provided + if(!sequence_reader.Open(arguments)) { - f_n++; - current_file = files[f_n]; - } - else - { - // If we want to write out from webcam - f_n = 0; - } - - // Do some grabbing - cv::VideoCapture video_capture; - if( current_file.size() > 0 ) - { - if (!boost::filesystem::exists(current_file)) + // If failed to open because no input files specified, attempt to open a webcam + if (sequence_reader.no_input_specified && sequence_number == 0) { - FATAL_STREAM("File does not exist"); - return 1; - } - - current_file = boost::filesystem::path(current_file).generic_string(); - - INFO_STREAM( "Attempting to read from file: " << current_file ); - video_capture = cv::VideoCapture( current_file ); - } - else - { - INFO_STREAM( "Attempting to capture from device: " << device ); - video_capture = cv::VideoCapture( device ); - - // Read a first frame often empty in camera - cv::Mat captured_image; - video_capture >> captured_image; - } - - if (!video_capture.isOpened()) - { - FATAL_STREAM("Failed to open video source"); - return 1; - } - else INFO_STREAM( "Device or file opened"); - - cv::Mat captured_image; - video_capture >> captured_image; - - // If optical centers are not defined just use center of image - if (cx_undefined) - { - cx = captured_image.cols / 2.0f; - cy = captured_image.rows / 2.0f; - } - // Use a rough guess-timate of focal length - if (fx_undefined) - { - fx = 500 * (captured_image.cols / 640.0); - fy = 500 * (captured_image.rows / 480.0); - - fx = (fx + fy) / 2.0; - fy = fx; - } - - int frame_count = 0; - - // saving the videos - cv::VideoWriter writerFace; - if (!output_video_files.empty()) - { - try - { - writerFace = cv::VideoWriter(output_video_files[f_n], CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), 30, captured_image.size(), true); - } - catch(cv::Exception e) - { - WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); - } - } - - // Use for timestamping if using a webcam - int64 t_initial = cv::getTickCount(); - - INFO_STREAM( "Starting tracking"); - while(!captured_image.empty()) - { - - // Reading the images - cv::Mat_ grayscale_image; - - if(captured_image.channels() == 3) - { - cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); + // If that fails, revert to webcam + INFO_STREAM("No input specified, attempting to open a webcam 0"); + if (!sequence_reader.OpenWebcam(0)) + { + ERROR_STREAM("Failed to open the webcam"); + break; + } } else { - grayscale_image = captured_image.clone(); + ERROR_STREAM("Failed to open a sequence"); + break; } - + } + INFO_STREAM("Device or file opened"); + + cv::Mat captured_image = sequence_reader.GetNextFrame(); + + INFO_STREAM("Starting tracking"); + while (!captured_image.empty()) // this is not a for loop as we might also be reading from a webcam + { + + // Reading the images + cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); + // The actual facial landmark detection / tracking - bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, clnf_model, det_parameters); - - // Visualising the results - // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised - double detection_certainty = clnf_model.detection_certainty; + bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters); // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); - if (det_parameters.track_gaze && detection_success && clnf_model.eye_model) + // If tracking succeeded and we have an eye model, estimate gaze + if (detection_success && face_model.eye_model) { - GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); - GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); + GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true); + GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false); } - visualise_tracking(captured_image, clnf_model, det_parameters, gazeDirection0, gazeDirection1, frame_count, fx, fy, cx, cy); - - // output the tracked video - if (!output_video_files.empty()) - { - writerFace << captured_image; - } + // Work out the pose of the head from the tracked model + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + // Keeping track of FPS + fps_tracker.AddFrame(); + + // Displaying the tracking visualizations + visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, detection_success); + visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); + visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); + visualizer.SetFps(fps_tracker.GetFPS()); + // detect key presses (due to pecularities of OpenCV, you can get it when displaying images) + char character_press = visualizer.ShowObservation(); - video_capture >> captured_image; - - // detect key presses - char character_press = cv::waitKey(1); - // restart the tracker - if(character_press == 'r') + if (character_press == 'r') { - clnf_model.Reset(); + face_model.Reset(); } // quit the application - else if(character_press=='q') + else if (character_press == 'q') { return(0); } - // Update the frame count - frame_count++; + // Grabbing the next frame in the sequence + captured_image = sequence_reader.GetNextFrame(); } - frame_count = 0; - // Reset the model, for the next video - clnf_model.Reset(); - - // break out of the loop if done with all the files (or using a webcam) - if(f_n == files.size() -1 || files.empty()) - { - done = true; - } - } + face_model.Reset(); + sequence_number++; + + } return 0; } diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj b/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj index 7283fdb..98ad4ba 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj @@ -112,7 +112,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false StreamingSIMDExtensions2 true @@ -128,7 +128,7 @@ Level3 Disabled WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false AdvancedVectorExtensions true @@ -147,7 +147,7 @@ true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false Speed StreamingSIMDExtensions2 @@ -164,16 +164,17 @@ Level3 NotUsing - MaxSpeed + Full - true + false WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false Speed AdvancedVectorExtensions true + AnySuitable Console @@ -195,6 +196,9 @@ {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} + + {8e741ea2-9386-4cf2-815e-6f9b08991eac} + diff --git a/exe/FaceLandmarkVidMulti/CMakeLists.txt b/exe/FaceLandmarkVidMulti/CMakeLists.txt index 3e4086c..87b51c4 100644 --- a/exe/FaceLandmarkVidMulti/CMakeLists.txt +++ b/exe/FaceLandmarkVidMulti/CMakeLists.txt @@ -5,9 +5,11 @@ include_directories(${TBB_ROOT_DIR}/include) include_directories(${LandmarkDetector_SOURCE_DIR}/include) include_directories(../../lib/local/LandmarkDetector/include) +include_directories(../../lib/local/Utilities/include) add_executable(FaceLandmarkVidMulti FaceLandmarkVidMulti.cpp) target_link_libraries(FaceLandmarkVidMulti LandmarkDetector) +target_link_libraries(FaceLandmarkVidMulti Utilities) target_link_libraries(FaceLandmarkVidMulti dlib) target_link_libraries(FaceLandmarkVidMulti ${OpenCV_LIBS} ${Boost_LIBRARIES} ${TBB_LIBRARIES} ${BLAS_LIBRARIES}) diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index 408dc37..7398fb2 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -36,6 +36,10 @@ // FaceTrackingVidMulti.cpp : Defines the entry point for the multiple face tracking console application. #include "LandmarkCoreIncludes.h" +#include "VisualizationUtils.h" +#include "Visualizer.h" +#include "SequenceCapture.h" + #include #include @@ -106,17 +110,8 @@ int main (int argc, char **argv) vector arguments = get_arguments(argc, argv); - // Some initial parameters that can be overriden from command line - vector files, tracked_videos_output, dummy_out; - - // By default try webcam 0 - int device = 0; - - // cx and cy aren't necessarilly in the image center, so need to be able to override it (start with unit vals and init them if none specified) - float fx = 600, fy = 600, cx = 0, cy = 0; LandmarkDetector::FaceModelParameters det_params(arguments); - det_params.use_face_template = true; // This is so that the model would not try re-initialising itself det_params.reinit_video_every = -1; @@ -124,114 +119,69 @@ int main (int argc, char **argv) vector det_parameters; det_parameters.push_back(det_params); - - // Get the input output file parameters - string output_codec; - LandmarkDetector::get_video_input_output_params(files, dummy_out, tracked_videos_output, output_codec, arguments); - - // Get camera parameters - LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments); // The modules that are being used for tracking - vector clnf_models; + vector face_models; vector active_models; int num_faces_max = 4; - LandmarkDetector::CLNF clnf_model(det_parameters[0].model_location); - clnf_model.face_detector_HAAR.load(det_parameters[0].face_detector_location); - clnf_model.face_detector_location = det_parameters[0].face_detector_location; + LandmarkDetector::CLNF face_model(det_parameters[0].model_location); + face_model.face_detector_HAAR.load(det_parameters[0].face_detector_location); + face_model.face_detector_location = det_parameters[0].face_detector_location; - clnf_models.reserve(num_faces_max); + face_models.reserve(num_faces_max); - clnf_models.push_back(clnf_model); + face_models.push_back(face_model); active_models.push_back(false); for (int i = 1; i < num_faces_max; ++i) { - clnf_models.push_back(clnf_model); + face_models.push_back(face_model); active_models.push_back(false); det_parameters.push_back(det_params); } - // If multiple video files are tracked, use this to indicate if we are done - bool done = false; - int f_n = -1; + // Open a sequence + Utilities::SequenceCapture sequence_reader; - // If cx (optical axis centre) is undefined will use the image size/2 as an estimate - bool cx_undefined = false; - if(cx == 0 || cy == 0) + // A utility for visualizing the results (show just the tracks) + Utilities::Visualizer visualizer(true, false, false); + + // Tracking FPS for visualization + Utilities::FpsTracker fps_tracker; + fps_tracker.AddFrame(); + + int sequence_number = 0; + + while(true) // this is not a for loop as we might also be reading from a webcam { - cx_undefined = true; - } - - while(!done) // this is not a for loop as we might also be reading from a webcam - { - - string current_file; - // We might specify multiple video files as arguments - if(files.size() > 0) + // The sequence reader chooses what to open based on command line arguments provided + if (!sequence_reader.Open(arguments)) { - f_n++; - current_file = files[f_n]; + // If failed to open because no input files specified, attempt to open a webcam + if (sequence_reader.no_input_specified && sequence_number == 0) + { + // If that fails, revert to webcam + INFO_STREAM("No input specified, attempting to open a webcam 0"); + if (!sequence_reader.OpenWebcam(0)) + { + ERROR_STREAM("Failed to open the webcam"); + break; + } + } + else + { + ERROR_STREAM("Failed to open a sequence"); + break; + } } + INFO_STREAM("Device or file opened"); - // Do some grabbing - cv::VideoCapture video_capture; - if( current_file.size() > 0 ) - { - INFO_STREAM( "Attempting to read from file: " << current_file ); - video_capture = cv::VideoCapture( current_file ); - } - else - { - INFO_STREAM( "Attempting to capture from device: " << device ); - video_capture = cv::VideoCapture( device ); + cv::Mat captured_image = sequence_reader.GetNextFrame(); - // Read a first frame often empty in camera - cv::Mat captured_image; - video_capture >> captured_image; - } - - if (!video_capture.isOpened()) - { - FATAL_STREAM("Failed to open video source"); - return 1; - } - else INFO_STREAM( "Device or file opened"); - - cv::Mat captured_image; - video_capture >> captured_image; - - - // If optical centers are not defined just use center of image - if(cx_undefined) - { - cx = captured_image.cols / 2.0f; - cy = captured_image.rows / 2.0f; - } - int frame_count = 0; - - // saving the videos - cv::VideoWriter writerFace; - if(!tracked_videos_output.empty()) - { - try - { - writerFace = cv::VideoWriter(tracked_videos_output[f_n], CV_FOURCC(output_codec[0],output_codec[1],output_codec[2],output_codec[3]), 30, captured_image.size(), true); - } - catch(cv::Exception e) - { - WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); - } - } - - // For measuring the timings - int64 t1,t0 = cv::getTickCount(); - double fps = 10; - INFO_STREAM( "Starting tracking"); while(!captured_image.empty()) @@ -254,7 +204,7 @@ int main (int argc, char **argv) vector > face_detections; bool all_models_active = true; - for(unsigned int model = 0; model < clnf_models.size(); ++model) + for(unsigned int model = 0; model < face_models.size(); ++model) { if(!active_models[model]) { @@ -268,33 +218,32 @@ int main (int argc, char **argv) if(det_parameters[0].curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) { vector confidences; - LandmarkDetector::DetectFacesHOG(face_detections, grayscale_image, clnf_models[0].face_detector_HOG, confidences); + LandmarkDetector::DetectFacesHOG(face_detections, grayscale_image, face_models[0].face_detector_HOG, confidences); } else { - LandmarkDetector::DetectFaces(face_detections, grayscale_image, clnf_models[0].face_detector_HAAR); + LandmarkDetector::DetectFaces(face_detections, grayscale_image, face_models[0].face_detector_HAAR); } } // Keep only non overlapping detections (also convert to a concurrent vector - NonOverlapingDetections(clnf_models, face_detections); + NonOverlapingDetections(face_models, face_detections); vector > face_detections_used(face_detections.size()); // Go through every model and update the tracking - tbb::parallel_for(0, (int)clnf_models.size(), [&](int model){ + tbb::parallel_for(0, (int)face_models.size(), [&](int model){ //for(unsigned int model = 0; model < clnf_models.size(); ++model) //{ bool detection_success = false; // If the current model has failed more than 4 times in a row, remove it - if(clnf_models[model].failures_in_a_row > 4) + if(face_models[model].failures_in_a_row > 4) { active_models[model] = false; - clnf_models[model].Reset(); - + face_models[model].Reset(); } // If the model is inactive reactivate it with new detections @@ -308,11 +257,11 @@ int main (int argc, char **argv) { // Reinitialise the model - clnf_models[model].Reset(); + face_models[model].Reset(); // This ensures that a wider window is used for the initial landmark localisation - clnf_models[model].detection_success = false; - detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_detections[detection_ind], clnf_models[model], det_parameters[model]); + face_models[model].detection_success = false; + detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_detections[detection_ind], face_models[model], det_parameters[model]); // This activates the model active_models[model] = true; @@ -326,123 +275,64 @@ int main (int argc, char **argv) else { // The actual facial landmark detection / tracking - detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, clnf_models[model], det_parameters[model]); + detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_models[model], det_parameters[model]); } }); + // Keeping track of FPS + fps_tracker.AddFrame(); + + visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + // Go through every model and visualise the results - for(size_t model = 0; model < clnf_models.size(); ++model) + for(size_t model = 0; model < face_models.size(); ++model) { // Visualising the results - // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised - double detection_certainty = clnf_models[model].detection_certainty; - - double visualisation_boundary = -0.1; - - // Only draw if the reliability is reasonable, the value is slightly ad-hoc - if(detection_certainty < visualisation_boundary) + if(active_models[model]) { - LandmarkDetector::Draw(disp_image, clnf_models[model]); - - if(detection_certainty > 1) - detection_certainty = 1; - if(detection_certainty < -1) - detection_certainty = -1; - - detection_certainty = (detection_certainty + 1)/(visualisation_boundary +1); - - // A rough heuristic for box around the face width - int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); - - // Work out the pose of the head from the tracked model - cv::Vec6d pose_estimate = LandmarkDetector::GetPose(clnf_models[model], fx, fy, cx, cy); - - // Draw it in reddish if uncertain, blueish if certain - LandmarkDetector::DrawBox(disp_image, pose_estimate, cv::Scalar((1-detection_certainty)*255.0,0, detection_certainty*255), thickness, fx, fy, cx, cy); + visualizer.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].detection_certainty, face_models[model].detection_success); + visualizer.SetObservationPose(LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty); } } + visualizer.SetFps(fps_tracker.GetFPS()); - // Work out the framerate - if(frame_count % 10 == 0) - { - t1 = cv::getTickCount(); - fps = 10.0 / (double(t1-t0)/cv::getTickFrequency()); - t0 = t1; - } - - // Write out the framerate on the image before displaying it - char fpsC[255]; - sprintf(fpsC, "%d", (int)fps); - string fpsSt("FPS:"); - fpsSt += fpsC; - cv::putText(disp_image, fpsSt, cv::Point(10,20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255,0,0), 1, CV_AA); - - int num_active_models = 0; - - for( size_t active_model = 0; active_model < active_models.size(); active_model++) - { - if(active_models[active_model]) - { - num_active_models++; - } - } - - char active_m_C[255]; - sprintf(active_m_C, "%d", num_active_models); - string active_models_st("Active models:"); - active_models_st += active_m_C; - cv::putText(disp_image, active_models_st, cv::Point(10,60), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255,0,0), 1, CV_AA); - - if(!det_parameters[0].quiet_mode) - { - cv::namedWindow("tracking_result",1); - cv::imshow("tracking_result", disp_image); - } - - // output the tracked video - if(!tracked_videos_output.empty()) - { - writerFace << disp_image; - } - - video_capture >> captured_image; - - // detect key presses - char character_press = cv::waitKey(1); + // show visualization and detect key presses + char character_press = visualizer.ShowObservation(); // restart the trackers if(character_press == 'r') { - for(size_t i=0; i < clnf_models.size(); ++i) + for(size_t i=0; i < face_models.size(); ++i) { - clnf_models[i].Reset(); + face_models[i].Reset(); active_models[i] = false; } } // quit the application else if(character_press=='q') { - return(0); + return 0; } // Update the frame count frame_count++; + + // Grabbing the next frame in the sequence + captured_image = sequence_reader.GetNextFrame(); + } frame_count = 0; // Reset the model, for the next video - for(size_t model=0; model < clnf_models.size(); ++model) + for(size_t model=0; model < face_models.size(); ++model) { - clnf_models[model].Reset(); + face_models[model].Reset(); active_models[model] = false; } - // break out of the loop if done with all the files - if(f_n == files.size() -1) - { - done = true; - } + sequence_number++; + } return 0; diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj index e8a84c6..d82f3dd 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj @@ -105,7 +105,7 @@ Level3 Disabled - $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 true @@ -117,7 +117,7 @@ Level3 Disabled - $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions true @@ -131,7 +131,7 @@ MaxSpeed true true - $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 MultiThreadedDLL true @@ -145,13 +145,15 @@ Level3 - MaxSpeed + Full true - true - $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + false + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions MultiThreadedDLL true + AnySuitable + Speed true @@ -166,6 +168,9 @@ {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} + + {8e741ea2-9386-4cf2-815e-6f9b08991eac} + diff --git a/exe/FeatureExtraction/CMakeLists.txt b/exe/FeatureExtraction/CMakeLists.txt index b3e1d18..39e379a 100644 --- a/exe/FeatureExtraction/CMakeLists.txt +++ b/exe/FeatureExtraction/CMakeLists.txt @@ -9,10 +9,12 @@ include_directories(${LandmarkDetector_SOURCE_DIR}/include) include_directories(../../lib/local/LandmarkDetector/include) include_directories(../../lib/local/FaceAnalyser/include) include_directories(../../lib/local/GazeAnalyser/include) +include_directories(../../lib/local/Utilities/include) target_link_libraries(FeatureExtraction LandmarkDetector) target_link_libraries(FeatureExtraction FaceAnalyser) target_link_libraries(FeatureExtraction GazeAnalyser) +target_link_libraries(FeatureExtraction Utilities) target_link_libraries(FeatureExtraction dlib) target_link_libraries(FeatureExtraction ${OpenCV_LIBS} ${Boost_LIBRARIES} ${TBB_LIBRARIES} ${BLAS_LIBRARIES}) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 08781d7..adc424f 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -56,6 +56,11 @@ #include #include #include +#include +#include +#include +#include +#include #ifndef CONFIG_DIR #define CONFIG_DIR "~" @@ -80,8 +85,6 @@ printErrorAndAbort( std::string( "Fatal error: " ) + stream ) using namespace std; -using namespace boost::filesystem; - vector get_arguments(int argc, char **argv) { @@ -95,1019 +98,157 @@ vector get_arguments(int argc, char **argv) return arguments; } -// Useful utility for creating directories for storing the output files -void create_directory_from_file(string output_path) -{ - - // Creating the right directory structure - - // First get rid of the file - auto p = path(path(output_path).parent_path()); - - if(!p.empty() && !boost::filesystem::exists(p)) - { - bool success = boost::filesystem::create_directories(p); - if(!success) - { - cout << "Failed to create a directory... " << p.string() << endl; - } - } -} - -void create_directory(string output_path) -{ - - // Creating the right directory structure - auto p = path(output_path); - - if(!boost::filesystem::exists(p)) - { - bool success = boost::filesystem::create_directories(p); - - if(!success) - { - cout << "Failed to create a directory..." << p.string() << endl; - } - } -} - -void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, bool& visualize_track, - bool& visualize_align, bool& visualize_hog, bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, - bool &output_pose, bool &output_AUs, bool &output_gaze, vector &arguments); - -void get_image_input_output_params_feats(vector > &input_image_files, bool& as_video, vector &arguments); - -void output_HOG_frame(std::ofstream* hog_file, bool good_frame, const cv::Mat_& hog_descriptor, int num_rows, int num_cols); - -// Some globals for tracking timing information for visualisation -double fps_tracker = -1.0; -int64 t0 = 0; - -// Visualising the results -void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, int frame_count, double fx, double fy, double cx, double cy) -{ - - // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised - double detection_certainty = face_model.detection_certainty; - bool detection_success = face_model.detection_success; - - double visualisation_boundary = 0.2; - - // Only draw if the reliability is reasonable, the value is slightly ad-hoc - if (detection_certainty < visualisation_boundary) - { - LandmarkDetector::Draw(captured_image, face_model); - - double vis_certainty = detection_certainty; - if (vis_certainty > 1) - vis_certainty = 1; - if (vis_certainty < -1) - vis_certainty = -1; - - vis_certainty = (vis_certainty + 1) / (visualisation_boundary + 1); - - // A rough heuristic for box around the face width - int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); - - cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy); - - // Draw it in reddish if uncertain, blueish if certain - LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar((1 - vis_certainty)*255.0, 0, vis_certainty * 255), thickness, fx, fy, cx, cy); - - if (det_parameters.track_gaze && detection_success && face_model.eye_model) - { - GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); - } - } - - // Work out the framerate - if (frame_count % 10 == 0) - { - double t1 = cv::getTickCount(); - fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency()); - t0 = t1; - } - - // Write out the framerate on the image before displaying it - char fpsC[255]; - std::sprintf(fpsC, "%d", (int)fps_tracker); - string fpsSt("FPS:"); - fpsSt += fpsC; - cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); - - if (!det_parameters.quiet_mode) - { - cv::namedWindow("tracking_result", 1); - cv::imshow("tracking_result", captured_image); - } -} - -void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, - bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector au_names_class, vector au_names_reg); - -// Output all of the information into one file in one go (quite a few parameters, but simplifies the flow) -void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, - bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success, - cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, - const FaceAnalysis::FaceAnalyser& face_analyser); - int main (int argc, char **argv) { vector arguments = get_arguments(argc, argv); - - // Some initial parameters that can be overriden from command line - vector input_files, output_files, tracked_videos_output; - - // Get the input output file parameters - - string output_codec; //not used but should - LandmarkDetector::get_video_input_output_params(input_files, output_files, tracked_videos_output, output_codec, arguments); - - bool video_input = true; - bool images_as_video = false; - - vector > input_image_files; - - // Adding image support for reading in the files - if(input_files.empty()) - { - vector d_files; - vector o_img; - vector> bboxes; - get_image_input_output_params_feats(input_image_files, images_as_video, arguments); - - if(!input_image_files.empty()) - { - video_input = false; - } - - } - - // Grab camera parameters, if they are not defined (approximate values will be used) - float fx = 0, fy = 0, cx = 0, cy = 0; - int d = 0; - // Get camera parameters - LandmarkDetector::get_camera_params(d, fx, fy, cx, cy, arguments); - - // If cx (optical axis centre) is undefined will use the image size/2 as an estimate - bool cx_undefined = false; - bool fx_undefined = false; - if (cx == 0 || cy == 0) - { - cx_undefined = true; - } - if (fx == 0 || fy == 0) - { - fx_undefined = true; - } - - vector output_similarity_align; - vector output_hog_align_files; - - // By default output all parameters, but these can be turned off to get smaller files or slightly faster processing times - // use -no2Dfp, -no3Dfp, -noMparams, -noPose, -noAUs, -noGaze to turn them off - bool output_2D_landmarks = true; - bool output_3D_landmarks = true; - bool output_model_params = true; - bool output_pose = true; - bool output_AUs = true; - bool output_gaze = true; - - bool visualize_track = false; - bool visualize_align = false; - bool visualize_hog = false; - get_output_feature_params(output_similarity_align, output_hog_align_files, visualize_track, visualize_align, visualize_hog, - output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, arguments); - - // If multiple video files are tracked, use this to indicate if we are done - bool done = false; - int f_n = -1; - int curr_img = -1; // Load the modules that are being used for tracking and face analysis // Load face landmark detector LandmarkDetector::FaceModelParameters det_parameters(arguments); // Always track gaze in feature extraction - det_parameters.track_gaze = true; LandmarkDetector::CLNF face_model(det_parameters.model_location); // Load facial feature extractor and AU analyser FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); - while(!done) // this is not a for loop as we might also be reading from a webcam + Utilities::SequenceCapture sequence_reader; + + // A utility for visualizing the results + Utilities::Visualizer visualizer(arguments); + + // Tracking FPS for visualization + Utilities::FpsTracker fps_tracker; + fps_tracker.AddFrame(); + + while (true) // this is not a for loop as we might also be reading from a webcam { + + // The sequence reader chooses what to open based on command line arguments provided + if(!sequence_reader.Open(arguments)) + break; + + INFO_STREAM("Device or file opened"); - string current_file; - - cv::VideoCapture video_capture; - + if (sequence_reader.IsWebcam()) + { + INFO_STREAM("WARNING: using a webcam in feature extraction, Action Unit predictions will not be as accurate in real-time webcam mode"); + INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)"); + visualizer.vis_track = true; + } + cv::Mat captured_image; - int total_frames = -1; - int reported_completion = 0; - - double fps_vid_in = -1.0; - - if(video_input) - { - // We might specify multiple video files as arguments - if(input_files.size() > 0) - { - f_n++; - current_file = input_files[f_n]; - } - else - { - // If we want to write out from webcam - f_n = 0; - } - // Do some grabbing - if( current_file.size() > 0 ) - { - INFO_STREAM( "Attempting to read from file: " << current_file ); - video_capture = cv::VideoCapture( current_file ); - total_frames = (int)video_capture.get(CV_CAP_PROP_FRAME_COUNT); - fps_vid_in = video_capture.get(CV_CAP_PROP_FPS); - - // Check if fps is nan or less than 0 - if (fps_vid_in != fps_vid_in || fps_vid_in <= 0) - { - INFO_STREAM("FPS of the video file cannot be determined, assuming 30"); - fps_vid_in = 30; - } - } - - if (!video_capture.isOpened()) - { - FATAL_STREAM("Failed to open video source, exiting"); - return 1; - } - else - { - INFO_STREAM("Device or file opened"); - } - - video_capture >> captured_image; - } - else - { - f_n++; - curr_img++; - if(!input_image_files[f_n].empty()) - { - string curr_img_file = input_image_files[f_n][curr_img]; - captured_image = cv::imread(curr_img_file, -1); - total_frames = input_image_files[f_n].size(); - } - else - { - FATAL_STREAM( "No .jpg or .png images in a specified drectory, exiting" ); - return 1; - } - - // If image sequence provided, assume the fps is 30 - fps_vid_in = 30; - } - // If optical centers are not defined just use center of image - if(cx_undefined) + Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(), + sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps); + Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); + + if (recording_params.outputGaze() && !face_model.eye_model) + cout << "WARNING: no eye model defined, but outputting gaze" << endl; + + captured_image = sequence_reader.GetNextFrame(); + + // For reporting progress + double reported_completion = 0; + + INFO_STREAM("Starting tracking"); + while (!captured_image.empty()) { - cx = captured_image.cols / 2.0f; - cy = captured_image.rows / 2.0f; - } - // Use a rough guess-timate of focal length - if (fx_undefined) - { - fx = 500 * (captured_image.cols / 640.0); - fy = 500 * (captured_image.rows / 480.0); - fx = (fx + fy) / 2.0; - fy = fx; - } - - // Creating output files - std::ofstream output_file; + // Converting to grayscale + cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); - if (!output_files.empty()) - { - output_file.open(output_files[f_n], ios_base::out); - int num_eye_landmarks = LandmarkDetector::CalculateAllEyeLandmarks(face_model).size(); - - prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), num_eye_landmarks, face_analyser.GetAUClassNames(), face_analyser.GetAURegNames()); - } - - // Saving the HOG features - std::ofstream hog_output_file; - if(!output_hog_align_files.empty()) - { - hog_output_file.open(output_hog_align_files[f_n], ios_base::out | ios_base::binary); - } - - // saving the videos - cv::VideoWriter writerFace; - if(!tracked_videos_output.empty()) - { - try - { - writerFace = cv::VideoWriter(tracked_videos_output[f_n], CV_FOURCC(output_codec[0],output_codec[1],output_codec[2],output_codec[3]), fps_vid_in, captured_image.size(), true); - } - catch(cv::Exception e) - { - WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); - } - - - } - - int frame_count = 0; - - // This is useful for a second pass run (if want AU predictions) - vector params_global_video; - vector successes_video; - vector> params_local_video; - vector> detected_landmarks_video; - - // Use for timestamping if using a webcam - int64 t_initial = cv::getTickCount(); - - // Timestamp in seconds of current processing - double time_stamp = 0; - - INFO_STREAM( "Starting tracking"); - while(!captured_image.empty()) - { - - // Grab the timestamp first - if (video_input) - { - time_stamp = (double)frame_count * (1.0 / fps_vid_in); - } - else - { - // if loading images assume 30fps - time_stamp = (double)frame_count * (1.0 / 30.0); - } - - // Reading the images - cv::Mat_ grayscale_image; - - if(captured_image.channels() == 3) - { - cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); - } - else - { - grayscale_image = captured_image.clone(); - } - // The actual facial landmark detection / tracking - bool detection_success; - - if(video_input || images_as_video) - { - detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters); - } - else - { - detection_success = LandmarkDetector::DetectLandmarksInImage(grayscale_image, face_model, det_parameters); - } - - // Gaze tracking, absolute gaze direction - cv::Point3f gazeDirection0(0, 0, -1); - cv::Point3f gazeDirection1(0, 0, -1); - cv::Vec2d gazeAngle(0, 0); + bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters); - if (det_parameters.track_gaze && detection_success && face_model.eye_model) + // Gaze tracking, absolute gaze direction + cv::Point3f gazeDirection0(0, 0, 0); cv::Point3f gazeDirection1(0, 0, 0); cv::Vec2d gazeAngle(0, 0); + + if (detection_success && face_model.eye_model) { - GazeAnalysis::EstimateGaze(face_model, gazeDirection0, fx, fy, cx, cy, true); - GazeAnalysis::EstimateGaze(face_model, gazeDirection1, fx, fy, cx, cy, false); + GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true); + GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false); gazeAngle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); } // Do face alignment cv::Mat sim_warped_img; - cv::Mat_ hog_descriptor; - int num_hog_rows, num_hog_cols; + cv::Mat_ hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0; - // But only if needed in output - if(!output_similarity_align.empty() || hog_output_file.is_open() || output_AUs) + // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization + if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog) { - face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, time_stamp, false, !det_parameters.quiet_mode && visualize_hog); + face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam()); face_analyser.GetLatestAlignedFace(sim_warped_img); - - if(!det_parameters.quiet_mode && visualize_align) - { - cv::imshow("sim_warp", sim_warped_img); - } - if(hog_output_file.is_open() || (visualize_hog && !det_parameters.quiet_mode)) - { - face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); - - if(visualize_hog && !det_parameters.quiet_mode) - { - cv::Mat_ hog_descriptor_vis; - FaceAnalysis::Visualise_FHOG(hog_descriptor, num_hog_rows, num_hog_cols, hog_descriptor_vis); - cv::imshow("hog", hog_descriptor_vis); - } - } + face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); } // Work out the pose of the head from the tracked model - cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy); + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); - if (hog_output_file.is_open()) + // Keeping track of FPS + fps_tracker.AddFrame(); + + // Displaying the tracking visualizations + visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + visualizer.SetObservationFaceAlign(sim_warped_img); + visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); + visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, detection_success); + visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); + visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); + visualizer.SetFps(fps_tracker.GetFPS()); + + // detect key presses + char character_press = visualizer.ShowObservation(); + + // quit processing the current sequence (useful when in Webcam mode) + if (character_press == 'q') { - output_HOG_frame(&hog_output_file, detection_success, hog_descriptor, num_hog_rows, num_hog_cols); + break; } - // Write the similarity normalised output - if (!output_similarity_align.empty()) - { - - char name[100]; - - // Filename is based on frame number - std::sprintf(name, "frame_det_%06d.bmp", frame_count + 1); - - // Construct the output filename - boost::filesystem::path slash("/"); - - std::string preferredSlash = slash.make_preferred().string(); - - string out_file = output_similarity_align[f_n] + preferredSlash + string(name); - bool write_success = imwrite(out_file, sim_warped_img); - - if (!write_success) - { - cout << "Could not output similarity aligned image image" << endl; - return 1; - } - } - - // Visualising the tracker - if(visualize_track && !det_parameters.quiet_mode) - { - visualise_tracking(captured_image, face_model, det_parameters, gazeDirection0, gazeDirection1, frame_count, fx, fy, cx, cy); - } - - // Output the landmarks, pose, gaze, parameters and AUs - outputAllFeatures(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, - face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1, gazeAngle, - pose_estimate, fx, fy, cx, cy, face_analyser); - - // output the tracked video - if(!tracked_videos_output.empty()) - { - writerFace << captured_image; - } - - if(video_input) - { - video_capture >> captured_image; - } - else - { - curr_img++; - if(curr_img < (int)input_image_files[f_n].size()) - { - string curr_img_file = input_image_files[f_n][curr_img]; - captured_image = cv::imread(curr_img_file, -1); - } - else - { - captured_image = cv::Mat(); - } - } + // Setting up the recorder output + open_face_rec.SetObservationHOG(detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG + open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); + open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); + open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), + face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success); + open_face_rec.SetObservationPose(pose_estimate); + open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy)); + open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp); + open_face_rec.SetObservationFaceAlign(sim_warped_img); + open_face_rec.WriteObservation(); - if (!det_parameters.quiet_mode) + // Reporting progress + if(sequence_reader.GetProgress() >= reported_completion / 10.0) { - // detect key presses - char character_press = cv::waitKey(1); - - // restart the tracker - if(character_press == 'r') + cout << reported_completion * 10 << "% "; + if (reported_completion == 10) { - face_model.Reset(); - } - // quit the application - else if(character_press=='q') - { - return(0); + cout << endl; } + reported_completion = reported_completion + 1; } - - // Update the frame count - frame_count++; - if(total_frames != -1) - { - if((double)frame_count/(double)total_frames >= reported_completion / 10.0) - { - cout << reported_completion * 10 << "% "; - reported_completion = reported_completion + 1; - } - } + // Grabbing the next frame in the sequence + captured_image = sequence_reader.GetNextFrame(); } - output_file.close(); + open_face_rec.Close(); - if (output_files.size() > 0 && output_AUs) + if (recording_params.outputAUs()) { - cout << "Postprocessing the Action Unit predictions" << endl; - face_analyser.PostprocessOutputFile(output_files[f_n]); + INFO_STREAM("Postprocessing the Action Unit predictions"); + face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile()); } + // Reset the models for the next video face_analyser.Reset(); face_model.Reset(); - frame_count = 0; - curr_img = -1; - - if (total_frames != -1) - { - cout << endl; - } - - // break out of the loop if done with all the files (or using a webcam) - if((video_input && f_n == input_files.size() -1) || (!video_input && f_n == input_image_files.size() - 1)) - { - done = true; - } } return 0; } - -void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, - bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - int num_face_landmarks, int num_model_modes, int num_eye_landmarks, vector au_names_class, vector au_names_reg) -{ - - *output_file << "frame, timestamp, confidence, success"; - - if (output_gaze) - { - *output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y"; - - for (int i = 0; i < num_eye_landmarks; ++i) - { - *output_file << ", eye_lmk_x_" << i; - } - for (int i = 0; i < num_eye_landmarks; ++i) - { - *output_file << ", eye_lmk_y_" << i; - } - } - - if (output_pose) - { - *output_file << ", pose_Tx, pose_Ty, pose_Tz, pose_Rx, pose_Ry, pose_Rz"; - } - - if (output_2D_landmarks) - { - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", x_" << i; - } - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", y_" << i; - } - } - - if (output_3D_landmarks) - { - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", X_" << i; - } - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", Y_" << i; - } - for (int i = 0; i < num_face_landmarks; ++i) - { - *output_file << ", Z_" << i; - } - } - - // Outputting model parameters (rigid and non-rigid), the first parameters are the 6 rigid shape parameters, they are followed by the non rigid shape parameters - if (output_model_params) - { - *output_file << ", p_scale, p_rx, p_ry, p_rz, p_tx, p_ty"; - for (int i = 0; i < num_model_modes; ++i) - { - *output_file << ", p_" << i; - } - } - - if (output_AUs) - { - std::sort(au_names_reg.begin(), au_names_reg.end()); - for (string reg_name : au_names_reg) - { - *output_file << ", " << reg_name << "_r"; - } - - std::sort(au_names_class.begin(), au_names_class.end()); - for (string class_name : au_names_class) - { - *output_file << ", " << class_name << "_c"; - } - } - - *output_file << endl; - -} - -// Output all of the information into one file in one go (quite a few parameters, but simplifies the flow) -void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, - bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success, - cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, - const FaceAnalysis::FaceAnalyser& face_analyser) -{ - - double confidence = 0.5 * (1 - face_model.detection_certainty); - - *output_file << frame_count + 1 << ", " << time_stamp << ", " << confidence << ", " << detection_success; - - // Output the estimated gaze - if (output_gaze) - { - *output_file << ", " << gazeDirection0.x << ", " << gazeDirection0.y << ", " << gazeDirection0.z - << ", " << gazeDirection1.x << ", " << gazeDirection1.y << ", " << gazeDirection1.z; - - // Output gaze angle (same format as head pose angle) - *output_file << ", " << gaze_angle[0] << ", " << gaze_angle[1]; - - // Output eye landmarks - std::vector eye_landmark_points = LandmarkDetector::CalculateAllEyeLandmarks(face_model); - - for (size_t i = 0; i < eye_landmark_points.size(); ++i) - { - *output_file << ", " << eye_landmark_points[i].x; - } - for (size_t i = 0; i < eye_landmark_points.size(); ++i) - { - *output_file << ", " << eye_landmark_points[i].y; - } - } - - // Output the estimated head pose - if (output_pose) - { - if(face_model.tracking_initialised) - { - *output_file << ", " << pose_estimate[0] << ", " << pose_estimate[1] << ", " << pose_estimate[2] - << ", " << pose_estimate[3] << ", " << pose_estimate[4] << ", " << pose_estimate[5]; - } - else - { - *output_file << ", 0, 0, 0, 0, 0, 0"; - } - } - - // Output the detected 2D facial landmarks - if (output_2D_landmarks) - { - for (int i = 0; i < face_model.pdm.NumberOfPoints() * 2; ++i) - { - if(face_model.tracking_initialised) - { - *output_file << ", " << face_model.detected_landmarks.at(i); - } - else - { - *output_file << ", 0"; - } - } - } - - // Output the detected 3D facial landmarks - if (output_3D_landmarks) - { - cv::Mat_ shape_3D = face_model.GetShape(fx, fy, cx, cy); - for (int i = 0; i < face_model.pdm.NumberOfPoints() * 3; ++i) - { - if (face_model.tracking_initialised) - { - *output_file << ", " << shape_3D.at(i); - } - else - { - *output_file << ", 0"; - } - } - } - - if (output_model_params) - { - for (int i = 0; i < 6; ++i) - { - if (face_model.tracking_initialised) - { - *output_file << ", " << face_model.params_global[i]; - } - else - { - *output_file << ", 0"; - } - } - for (int i = 0; i < face_model.pdm.NumberOfModes(); ++i) - { - if(face_model.tracking_initialised) - { - *output_file << ", " << face_model.params_local.at(i, 0); - } - else - { - *output_file << ", 0"; - } - } - } - - - - if (output_AUs) - { - auto aus_reg = face_analyser.GetCurrentAUsReg(); - - vector au_reg_names = face_analyser.GetAURegNames(); - std::sort(au_reg_names.begin(), au_reg_names.end()); - - // write out ar the correct index - for (string au_name : au_reg_names) - { - for (auto au_reg : aus_reg) - { - if (au_name.compare(au_reg.first) == 0) - { - *output_file << ", " << au_reg.second; - break; - } - } - } - - if (aus_reg.size() == 0) - { - for (size_t p = 0; p < face_analyser.GetAURegNames().size(); ++p) - { - *output_file << ", 0"; - } - } - - auto aus_class = face_analyser.GetCurrentAUsClass(); - - vector au_class_names = face_analyser.GetAUClassNames(); - std::sort(au_class_names.begin(), au_class_names.end()); - - // write out ar the correct index - for (string au_name : au_class_names) - { - for (auto au_class : aus_class) - { - if (au_name.compare(au_class.first) == 0) - { - *output_file << ", " << au_class.second; - break; - } - } - } - - if (aus_class.size() == 0) - { - for (size_t p = 0; p < face_analyser.GetAUClassNames().size(); ++p) - { - *output_file << ", 0"; - } - } - } - *output_file << endl; -} - - -void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, bool& visualize_track, - bool& visualize_align, bool& visualize_hog, bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, - bool &output_pose, bool &output_AUs, bool &output_gaze, vector &arguments) -{ - output_similarity_aligned.clear(); - output_hog_aligned_files.clear(); - - bool* valid = new bool[arguments.size()]; - - for (size_t i = 0; i < arguments.size(); ++i) - { - valid[i] = true; - } - - string output_root = ""; - - visualize_align = false; - visualize_hog = false; - visualize_track = false; - - string separator = string(1, boost::filesystem::path::preferred_separator); - - // First check if there is a root argument (so that videos and outputs could be defined more easilly) - for (size_t i = 0; i < arguments.size(); ++i) - { - if (arguments[i].compare("-root") == 0) - { - output_root = arguments[i + 1] + separator; - i++; - } - if (arguments[i].compare("-outroot") == 0) - { - output_root = arguments[i + 1] + separator; - i++; - } - } - - for (size_t i = 0; i < arguments.size(); ++i) - { - if (arguments[i].compare("-simalign") == 0) - { - output_similarity_aligned.push_back(output_root + arguments[i + 1]); - create_directory(output_root + arguments[i + 1]); - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-hogalign") == 0) - { - output_hog_aligned_files.push_back(output_root + arguments[i + 1]); - create_directory_from_file(output_root + arguments[i + 1]); - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-verbose") == 0) - { - visualize_track = true; - visualize_align = true; - visualize_hog = true; - } - else if (arguments[i].compare("-vis-align") == 0) - { - visualize_align = true; - valid[i] = false; - } - else if (arguments[i].compare("-vis-hog") == 0) - { - visualize_hog = true; - valid[i] = false; - } - else if (arguments[i].compare("-vis-track") == 0) - { - visualize_track = true; - valid[i] = false; - } - else if (arguments[i].compare("-no2Dfp") == 0) - { - output_2D_landmarks = false; - valid[i] = false; - } - else if (arguments[i].compare("-no3Dfp") == 0) - { - output_3D_landmarks = false; - valid[i] = false; - } - else if (arguments[i].compare("-noMparams") == 0) - { - output_model_params = false; - valid[i] = false; - } - else if (arguments[i].compare("-noPose") == 0) - { - output_pose = false; - valid[i] = false; - } - else if (arguments[i].compare("-noAUs") == 0) - { - output_AUs = false; - valid[i] = false; - } - else if (arguments[i].compare("-noGaze") == 0) - { - output_gaze = false; - valid[i] = false; - } - } - - for (int i = arguments.size() - 1; i >= 0; --i) - { - if (!valid[i]) - { - arguments.erase(arguments.begin() + i); - } - } - -} - - -// Can process images via directories creating a separate output file per directory -void get_image_input_output_params_feats(vector > &input_image_files, bool& as_video, vector &arguments) -{ - bool* valid = new bool[arguments.size()]; - - for (size_t i = 0; i < arguments.size(); ++i) - { - valid[i] = true; - if (arguments[i].compare("-fdir") == 0) - { - - // parse the -fdir directory by reading in all of the .png and .jpg files in it - path image_directory(arguments[i + 1]); - - try - { - // does the file exist and is it a directory - if (exists(image_directory) && is_directory(image_directory)) - { - - vector file_in_directory; - copy(directory_iterator(image_directory), directory_iterator(), back_inserter(file_in_directory)); - - // Sort the images in the directory first - sort(file_in_directory.begin(), file_in_directory.end()); - - vector curr_dir_files; - - for (vector::const_iterator file_iterator(file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator) - { - // Possible image extension .jpg and .png - if (file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".png") == 0) - { - curr_dir_files.push_back(file_iterator->string()); - } - } - - input_image_files.push_back(curr_dir_files); - } - } - catch (const filesystem_error& ex) - { - cout << ex.what() << '\n'; - } - - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-asvid") == 0) - { - as_video = true; - } - } - - // Clear up the argument list - for (int i = arguments.size() - 1; i >= 0; --i) - { - if (!valid[i]) - { - arguments.erase(arguments.begin() + i); - } - } - -} - -void output_HOG_frame(std::ofstream* hog_file, bool good_frame, const cv::Mat_& hog_descriptor, int num_rows, int num_cols) -{ - - // Using FHOGs, hence 31 channels - int num_channels = 31; - - hog_file->write((char*)(&num_cols), 4); - hog_file->write((char*)(&num_rows), 4); - hog_file->write((char*)(&num_channels), 4); - - // Not the best way to store a bool, but will be much easier to read it - float good_frame_float; - if (good_frame) - good_frame_float = 1; - else - good_frame_float = -1; - - hog_file->write((char*)(&good_frame_float), 4); - - cv::MatConstIterator_ descriptor_it = hog_descriptor.begin(); - - for (int y = 0; y < num_cols; ++y) - { - for (int x = 0; x < num_rows; ++x) - { - for (unsigned int o = 0; o < 31; ++o) - { - - float hog_data = (float)(*descriptor_it++); - hog_file->write((char*)&hog_data, 4); - } - } - } -} diff --git a/exe/FeatureExtraction/FeatureExtraction.vcxproj b/exe/FeatureExtraction/FeatureExtraction.vcxproj index 12df2bc..cc9be52 100644 --- a/exe/FeatureExtraction/FeatureExtraction.vcxproj +++ b/exe/FeatureExtraction/FeatureExtraction.vcxproj @@ -111,7 +111,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false StreamingSIMDExtensions2 true @@ -127,7 +127,7 @@ Level3 Disabled WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false AdvancedVectorExtensions true @@ -145,7 +145,7 @@ false true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false Speed StreamingSIMDExtensions2 @@ -163,16 +163,17 @@ Level3 NotUsing - MaxSpeed + Full false - true + false WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories) false Speed AdvancedVectorExtensions MultiThreadedDLL true + AnySuitable Console @@ -194,6 +195,9 @@ {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} + + {8e741ea2-9386-4cf2-815e-6f9b08991eac} + diff --git a/exe/Recording/Recording.vcxproj b/exe/Recording/Recording.vcxproj index 5f04e4c..ffd4bbd 100644 --- a/exe/Recording/Recording.vcxproj +++ b/exe/Recording/Recording.vcxproj @@ -86,6 +86,7 @@ Level3 Disabled true + ./include;%(AdditionalIncludeDirectories) true @@ -110,6 +111,7 @@ false Cdecl true + ./include;%(AdditionalIncludeDirectories) true @@ -128,6 +130,7 @@ false Cdecl true + ./include;%(AdditionalIncludeDirectories) true diff --git a/exe/releases/package_windows_executables.m b/exe/releases/package_windows_executables.m new file mode 100644 index 0000000..e8a8593 --- /dev/null +++ b/exe/releases/package_windows_executables.m @@ -0,0 +1,78 @@ +clear; +version = '0.3.0'; + +out_x86 = sprintf('OpenFace_%s_win_x86', version); +out_x64 = sprintf('OpenFace_%s_win_x64', version); + +mkdir(out_x86); +mkdir(out_x64); + +in_x86 = '../../Release/'; +in_x64 = '../../x64/Release/'; + +% Copy models +copyfile([in_x86, 'AU_predictors'], [out_x86, '/AU_predictors']) +copyfile([in_x86, 'classifiers'], [out_x86, '/classifiers']) +copyfile([in_x86, 'model'], [out_x86, '/model']) + +copyfile([in_x64, 'AU_predictors'], [out_x64, '/AU_predictors']) +copyfile([in_x64, 'classifiers'], [out_x64, '/classifiers']) +copyfile([in_x64, 'model'], [out_x64, '/model']) + +%% Copy libraries +libs_x86 = dir([in_x86, '*.lib'])'; + +for lib = libs_x86 + + copyfile([in_x86, '/', lib.name], [out_x86, '/', lib.name]) + +end + +libs_x64 = dir([in_x64, '*.lib'])'; + +for lib = libs_x64 + + copyfile([in_x64, '/', lib.name], [out_x64, '/', lib.name]) + +end + +%% Copy dlls +dlls_x86 = dir([in_x86, '*.dll'])'; + +for dll = dlls_x86 + + copyfile([in_x86, '/', dll.name], [out_x86, '/', dll.name]) + +end + +dlls_x64 = dir([in_x64, '*.dll'])'; + +for dll = dlls_x64 + + copyfile([in_x64, '/', dll.name], [out_x64, '/', dll.name]) + +end + +%% Copy exe's +exes_x86 = dir([in_x86, '*.exe'])'; + +for exe = exes_x86 + + copyfile([in_x86, '/', exe.name], [out_x86, '/', exe.name]) + +end + +exes_x64 = dir([in_x64, '*.exe'])'; + +for exe = exes_x64 + + copyfile([in_x64, '/', exe.name], [out_x64, '/', exe.name]) + +end + +%% Copy license and copyright +copyfile('../../Copyright.txt', [out_x86, '/Copyright.txt']); +copyfile('../../OpenFace-license.txt', [out_x86, '/OpenFace-license.txt']); + +copyfile('../../Copyright.txt', [out_x64, '/Copyright.txt']); +copyfile('../../OpenFace-license.txt', [out_x64, '/OpenFace-license.txt']); diff --git a/imgs/landmark_scheme_68.png b/imgs/landmark_scheme_68.png new file mode 100644 index 0000000..215fa6f Binary files /dev/null and b/imgs/landmark_scheme_68.png differ diff --git a/imgs/muticomp_logo_black.png b/imgs/muticomp_logo_black.png new file mode 100644 index 0000000..14004e4 Binary files /dev/null and b/imgs/muticomp_logo_black.png differ diff --git a/imgs/muticomp_logo_white.png b/imgs/muticomp_logo_white.png new file mode 100644 index 0000000..8e40748 Binary files /dev/null and b/imgs/muticomp_logo_white.png differ diff --git a/imgs/rainbow-logo.gif b/imgs/rainbow-logo.gif new file mode 100644 index 0000000..c8283a1 Binary files /dev/null and b/imgs/rainbow-logo.gif differ diff --git a/lib/local/CppInerop/FaceAnalyserInterop.h b/lib/local/CppInerop/FaceAnalyserInterop.h index de86d2f..5603f62 100644 --- a/lib/local/CppInerop/FaceAnalyserInterop.h +++ b/lib/local/CppInerop/FaceAnalyserInterop.h @@ -187,7 +187,7 @@ public: face_analyser->PostprocessOutputFile(msclr::interop::marshal_as(file)); } - void AddNextFrame(OpenCVWrappers::RawImage^ frame, List^>^ landmarks, bool success, bool online, bool vis_hog) { + void AddNextFrame(OpenCVWrappers::RawImage^ frame, List^>^ landmarks, bool success, bool online) { // Construct an OpenCV matric from the landmarks cv::Mat_ landmarks_mat(landmarks->Count * 2, 1, 0.0); @@ -197,7 +197,9 @@ public: landmarks_mat.at(i + landmarks->Count, 0) = landmarks[i]->Item2; } - face_analyser->AddNextFrame(frame->Mat, landmarks_mat, success, 0, online, vis_hog); + //(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam()); + + face_analyser->AddNextFrame(frame->Mat, landmarks_mat, success, 0, online); face_analyser->GetLatestHOG(*hog_features, *num_rows, *num_cols); diff --git a/lib/local/CppInerop/LandmarkDetectorInterop.h b/lib/local/CppInerop/LandmarkDetectorInterop.h index af7e339..b69982b 100644 --- a/lib/local/CppInerop/LandmarkDetectorInterop.h +++ b/lib/local/CppInerop/LandmarkDetectorInterop.h @@ -97,7 +97,6 @@ namespace CppInterop { params->model_location = "model/main_clnf_demos.txt"; } - params->track_gaze = true; } // TODO this could have optimize for demo mode (also could appropriately update sigma, reg_factor as well) diff --git a/lib/local/FaceAnalyser/CMakeLists.txt b/lib/local/FaceAnalyser/CMakeLists.txt index 70ea739..2e31f25 100644 --- a/lib/local/FaceAnalyser/CMakeLists.txt +++ b/lib/local/FaceAnalyser/CMakeLists.txt @@ -10,6 +10,9 @@ include_directories(../../3rdParty/OpenBLAS/include) #LandmarkDetector library include_directories(../../local/LandmarkDetector/include) +#Utilities library +include_directories(../../local/Utilities/include) + SET(SOURCE src/Face_utils.cpp src/FaceAnalyser.cpp diff --git a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj index 061e0e1..8e04272 100644 --- a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj +++ b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj @@ -98,7 +98,7 @@ Level3 Disabled false - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 true @@ -117,10 +117,9 @@ Level3 Disabled false - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions true - WIN64;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) true @@ -140,7 +139,7 @@ true - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 true @@ -160,15 +159,16 @@ Level3 - MaxSpeed + Full true - true + false - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions true - WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) + AnySuitable + Speed true @@ -187,6 +187,9 @@ {b47a5f12-2567-44e9-ae49-35763ec82149} + + {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} + diff --git a/lib/local/FaceAnalyser/include/FaceAnalyser.h b/lib/local/FaceAnalyser/include/FaceAnalyser.h index cae0b22..4a66d88 100644 --- a/lib/local/FaceAnalyser/include/FaceAnalyser.h +++ b/lib/local/FaceAnalyser/include/FaceAnalyser.h @@ -64,9 +64,7 @@ public: // Constructor for FaceAnalyser using the parameters structure FaceAnalyser(const FaceAnalysis::FaceAnalyserParameters& face_analyser_params); - void AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online = false, bool visualise = true); - - cv::Mat GetLatestHOGDescriptorVisualisation(); + void AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online = false); double GetCurrentTimeSeconds(); @@ -75,9 +73,8 @@ public: std::vector> GetCurrentAUsReg() const; // AU intensity std::vector> GetCurrentAUsCombined() const; // Both presense and intensity - // A standalone call for predicting AUs from a static image, the first element in the pair represents occurence the second intensity - // This call is useful for detecting action units in images - std::pair>, std::vector>> PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise = true); + // A standalone call for predicting AUs and computing face texture features from a static image + void PredictStaticAUsAndComputeFeatures(const cv::Mat& frame, const cv::Mat_& detected_landmarks); void Reset(); @@ -132,7 +129,6 @@ private: // Cache of intermediate images cv::Mat aligned_face_for_au; cv::Mat aligned_face_for_output; - cv::Mat hog_descriptor_visualisation; bool out_grayscale; // Private members to be used for predictions diff --git a/lib/local/FaceAnalyser/include/Face_utils.h b/lib/local/FaceAnalyser/include/Face_utils.h index 8e4439c..759bdb9 100644 --- a/lib/local/FaceAnalyser/include/Face_utils.h +++ b/lib/local/FaceAnalyser/include/Face_utils.h @@ -52,8 +52,6 @@ namespace FaceAnalysis void Extract_FHOG_descriptor(cv::Mat_& descriptor, const cv::Mat& image, int& num_rows, int& num_cols, int cell_size = 8); - void Visualise_FHOG(const cv::Mat_& descriptor, int num_rows, int num_cols, cv::Mat& visualisation); - // The following two methods go hand in hand void ExtractSummaryStatistics(const cv::Mat_& descriptors, cv::Mat_& sum_stats, bool mean, bool stdev, bool max_min); void AddDescriptor(cv::Mat_& descriptors, cv::Mat_ new_descriptor, int curr_frame, int num_frames_to_keep = 120); @@ -70,26 +68,10 @@ namespace FaceAnalysis cv::Matx22f AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst); //=========================================================================== - // Visualisation functions + // Visualisation functions, TODO move //=========================================================================== void Project(cv::Mat_& dest, const cv::Mat_& mesh, float fx, float fy, float cx, float cy); - //=========================================================================== - // Angle representation conversion helpers - //=========================================================================== - cv::Matx33f Euler2RotationMatrix(const cv::Vec3f& eulerAngles); - - // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign - cv::Vec3f RotationMatrix2Euler(const cv::Matx33f& rotation_matrix); - - cv::Vec3f Euler2AxisAngle(const cv::Vec3f& euler); - - cv::Vec3f AxisAngle2Euler(const cv::Vec3f& axis_angle); - - cv::Matx33f AxisAngle2RotationMatrix(const cv::Vec3f& axis_angle); - - cv::Vec3f RotationMatrix2AxisAngle(const cv::Matx33f& rotation_matrix); - //============================================================================ // Matrix reading functionality //============================================================================ diff --git a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp index 3702537..e262809 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp @@ -248,7 +248,7 @@ int GetViewId(const vector orientations_all, const cv::Vec3d& orienta } -std::pair>, std::vector>> FaceAnalyser::PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise) +void FaceAnalyser::PredictStaticAUsAndComputeFeatures(const cv::Mat& frame, const cv::Mat_& detected_landmarks) { // Extract shape parameters from the detected landmarks @@ -259,6 +259,16 @@ std::pair>, std::vector hog_descriptor; Extract_FHOG_descriptor(hog_descriptor, aligned_face_for_au, this->num_hog_rows, this->num_hog_cols); @@ -285,13 +295,7 @@ std::pair>, std::vector aligned_face_cols(1, aligned_face_for_au.cols * aligned_face_for_au.rows * aligned_face_for_au.channels(), aligned_face_for_au.data, 1); //cv::Mat_ aligned_face_cols_double; //aligned_face_cols.convertTo(aligned_face_cols_double, CV_64F); - - // Visualising the median HOG - if (visualise) - { - FaceAnalysis::Visualise_FHOG(hog_descriptor, num_hog_rows, num_hog_cols, hog_descriptor_visualisation); - } - + // Perform AU prediction auto AU_predictions_intensity = PredictCurrentAUs(orientation_to_use); auto AU_predictions_occurence = PredictCurrentAUsClass(orientation_to_use); @@ -305,12 +309,13 @@ std::pair>, std::vector 5) AU_predictions_intensity[au].second = 5; } - - return std::pair>, std::vector>>(AU_predictions_intensity, AU_predictions_occurence); + + AU_predictions_reg = AU_predictions_intensity; + AU_predictions_class = AU_predictions_occurence; } -void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online, bool visualise) +void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online) { frames_tracking++; @@ -318,12 +323,13 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& det // Extract shape parameters from the detected landmarks cv::Vec6f params_global; cv::Mat_ params_local; - pdm.CalcParams(params_global, params_local, detected_landmarks); // First align the face if tracking was successfull if(success) { + pdm.CalcParams(params_global, params_local, detected_landmarks); + // The aligned face requirement for AUs AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_au, align_width_au, align_height_au); @@ -343,6 +349,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& det aligned_face_for_au = cv::Mat(align_height_au, align_width_au, CV_8UC3); aligned_face_for_output.setTo(0); aligned_face_for_au.setTo(0); + params_local = cv::Mat_(pdm.NumberOfModes(), 1, 0.0f); } if (aligned_face_for_output.channels() == 3 && out_grayscale) @@ -423,22 +430,10 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& det { UpdateRunningMedian(this->geom_desc_hist, this->geom_hist_sum, this->geom_descriptor_median, geom_descriptor_frame, update_median, this->num_bins_geom, this->min_val_geom, this->max_val_geom); } - - // Visualising the median HOG - if(visualise) - { - FaceAnalysis::Visualise_FHOG(hog_descriptor, num_hog_rows, num_hog_cols, hog_descriptor_visualisation); - } - + // Perform AU prediction AU_predictions_reg = PredictCurrentAUs(orientation_to_use); - std::vector> AU_predictions_reg_corrected; - if(online) - { - AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, success, true); - } - // Add the reg predictions to the historic data for (size_t au = 0; au < AU_predictions_reg.size(); ++au) { @@ -452,6 +447,9 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& det else { AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(0); + + // Also invalidate AU if not successful + AU_predictions_reg[au].second = 0; } } @@ -469,22 +467,26 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& det else { AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(0); - } - } - - if(online) + // Also invalidate AU if not successful + AU_predictions_class[au].second = 0; + } + } + + // A workaround for online predictions to make them a bit more accurate + std::vector> AU_predictions_reg_corrected; + if (online) { + AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, success, true); AU_predictions_reg = AU_predictions_reg_corrected; } - else + + // Useful for prediction corrections (calibration after the whole video is processed) + if (success && frames_tracking_succ - 1 < max_init_frames) { - if (success && frames_tracking_succ - 1 < max_init_frames) - { - hog_desc_frames_init.push_back(hog_descriptor); - geom_descriptor_frames_init.push_back(geom_descriptor_frame); - views.push_back(orientation_to_use); - } + hog_desc_frames_init.push_back(hog_descriptor); + geom_descriptor_frames_init.push_back(geom_descriptor_frame); + views.push_back(orientation_to_use); } this->current_time_seconds = timestamp_seconds; @@ -982,11 +984,6 @@ vector> FaceAnalyser::PredictCurrentAUsClass(int view) return predictions; } -cv::Mat FaceAnalyser::GetLatestHOGDescriptorVisualisation() -{ - return hog_descriptor_visualisation; -} - vector> FaceAnalyser::GetCurrentAUsClass() const { return AU_predictions_class; @@ -1324,7 +1321,10 @@ void FaceAnalyser::PostprocessOutputFile(string output_file) // Now overwrite the whole file std::ofstream outfile(output_file, ios_base::out); // Write the header - outfile << std::setprecision(4); + outfile << std::setprecision(2); + outfile << std::fixed; + outfile << std::noshowpoint; + outfile << output_file_contents[0].c_str() << endl; // Write the contents diff --git a/lib/local/FaceAnalyser/src/Face_utils.cpp b/lib/local/FaceAnalyser/src/Face_utils.cpp index 2acfec9..f2cfe25 100644 --- a/lib/local/FaceAnalyser/src/Face_utils.cpp +++ b/lib/local/FaceAnalyser/src/Face_utils.cpp @@ -242,30 +242,6 @@ namespace FaceAnalysis } } - - void Visualise_FHOG(const cv::Mat_& descriptor, int num_rows, int num_cols, cv::Mat& visualisation) - { - - // First convert to dlib format - dlib::array2d > hog(num_rows, num_cols); - - cv::MatConstIterator_ descriptor_it = descriptor.begin(); - for(int y = 0; y < num_cols; ++y) - { - for(int x = 0; x < num_rows; ++x) - { - for(unsigned int o = 0; o < 31; ++o) - { - hog[y][x](o) = *descriptor_it++; - } - } - } - - // Draw the FHOG to OpenCV format - auto fhog_vis = dlib::draw_fhog(hog); - visualisation = dlib::toMat(fhog_vis).clone(); - } - // Create a row vector Felzenszwalb HOG descriptor from a given image void Extract_FHOG_descriptor(cv::Mat_& descriptor, const cv::Mat& image, int& num_rows, int& num_cols, int cell_size) { @@ -442,7 +418,7 @@ namespace FaceAnalysis //=========================================================================== - // Visualisation functions + // Visualisation functions, TODO rem //=========================================================================== void Project(cv::Mat_& dest, const cv::Mat_& mesh, float fx, float fy, float cx, float cy) { @@ -485,81 +461,6 @@ namespace FaceAnalysis } - //=========================================================================== - // Angle representation conversion helpers - //=========================================================================== - - // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign - cv::Matx33f Euler2RotationMatrix(const cv::Vec3f& eulerAngles) - { - cv::Matx33f rotation_matrix; - - float s1 = sin(eulerAngles[0]); - float s2 = sin(eulerAngles[1]); - float s3 = sin(eulerAngles[2]); - - float c1 = cos(eulerAngles[0]); - float c2 = cos(eulerAngles[1]); - float c3 = cos(eulerAngles[2]); - - rotation_matrix(0, 0) = c2 * c3; - rotation_matrix(0, 1) = -c2 *s3; - rotation_matrix(0, 2) = s2; - rotation_matrix(1, 0) = c1 * s3 + c3 * s1 * s2; - rotation_matrix(1, 1) = c1 * c3 - s1 * s2 * s3; - rotation_matrix(1, 2) = -c2 * s1; - rotation_matrix(2, 0) = s1 * s3 - c1 * c3 * s2; - rotation_matrix(2, 1) = c3 * s1 + c1 * s2 * s3; - rotation_matrix(2, 2) = c1 * c2; - - return rotation_matrix; - } - - // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign - cv::Vec3f RotationMatrix2Euler(const cv::Matx33f& rotation_matrix) - { - float q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0f; - float q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0f*q0); - float q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0f*q0); - float q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0f*q0); - - float t1 = 2.0f * (q0*q2 + q1*q3); - - float yaw = asin(2.0 * (q0*q2 + q1*q3)); - float pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3); - float roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3); - - return cv::Vec3f(pitch, yaw, roll); - } - - cv::Vec3f Euler2AxisAngle(const cv::Vec3f& euler) - { - cv::Matx33f rotMatrix = Euler2RotationMatrix(euler); - cv::Vec3f axis_angle; - cv::Rodrigues(rotMatrix, axis_angle); - return axis_angle; - } - - cv::Vec3f AxisAngle2Euler(const cv::Vec3f& axis_angle) - { - cv::Matx33f rotation_matrix; - cv::Rodrigues(axis_angle, rotation_matrix); - return RotationMatrix2Euler(rotation_matrix); - } - - cv::Matx33f AxisAngle2RotationMatrix(const cv::Vec3f& axis_angle) - { - cv::Matx33f rotation_matrix; - cv::Rodrigues(axis_angle, rotation_matrix); - return rotation_matrix; - } - - cv::Vec3f RotationMatrix2AxisAngle(const cv::Matx33f& rotation_matrix) - { - cv::Vec3f axis_angle; - cv::Rodrigues(rotation_matrix, axis_angle); - return axis_angle; - } //============================================================================ // Matrix reading functionality diff --git a/lib/local/FaceAnalyser/src/PDM.cpp b/lib/local/FaceAnalyser/src/PDM.cpp index 7fc6c20..f6533e4 100644 --- a/lib/local/FaceAnalyser/src/PDM.cpp +++ b/lib/local/FaceAnalyser/src/PDM.cpp @@ -48,6 +48,7 @@ #endif #include +#include // OpenBLAS #include @@ -107,7 +108,7 @@ void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_ // get the rotation matrix from the euler angles cv::Vec3f euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33f currRot = Euler2RotationMatrix(euler); + cv::Matx33f currRot = Utilities::Euler2RotationMatrix(euler); // get the 3D shape of the object cv::Mat_ Shape_3D = mean_shape + princ_comp * params_local; @@ -138,7 +139,7 @@ void PDM::CalcParams(cv::Vec6f& out_params_global, const cv::Rect_& bound CalcShape3D(current_shape, params_local); // rotate the shape - cv::Matx33f rotation_matrix = Euler2RotationMatrix(rotation); + cv::Matx33f rotation_matrix = Utilities::Euler2RotationMatrix(rotation); cv::Mat_ reshaped = current_shape.reshape(1, 3); @@ -213,7 +214,7 @@ void PDM::ComputeRigidJacobian(const cv::Mat_& p_local, const cv::Vec6f& // Get the rotation matrix cv::Vec3f euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33f currRot = Euler2RotationMatrix(euler); + cv::Matx33f currRot = Utilities::Euler2RotationMatrix(euler); float r11 = currRot(0, 0); float r12 = currRot(0, 1); @@ -306,7 +307,7 @@ void PDM::ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6f& this->CalcShape3D(shape_3D, params_local); cv::Vec3f euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33f currRot = Euler2RotationMatrix(euler); + cv::Matx33f currRot = Utilities::Euler2RotationMatrix(euler); float r11 = currRot(0, 0); float r12 = currRot(0, 1); @@ -404,7 +405,7 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& // get the original rotation matrix cv::Vec3f eulerGlobal(params_global[1], params_global[2], params_global[3]); - cv::Matx33f R1 = Euler2RotationMatrix(eulerGlobal); + cv::Matx33f R1 = Utilities::Euler2RotationMatrix(eulerGlobal); // construct R' = [1, -wz, wy // wz, 1, -wx @@ -422,8 +423,8 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& cv::Matx33f R3 = R1 *R2; // Extract euler angle (through axis angle first to make sure it's legal) - cv::Vec3f axis_angle = RotationMatrix2AxisAngle(R3); - cv::Vec3f euler = AxisAngle2Euler(axis_angle); + cv::Vec3f axis_angle = Utilities::RotationMatrix2AxisAngle(R3); + cv::Vec3f euler = Utilities::AxisAngle2Euler(axis_angle); params_global[1] = euler[0]; params_global[2] = euler[1]; @@ -466,7 +467,7 @@ void PDM::CalcParams(cv::Vec6f& out_params_global, cv::Mat_& out_params_l float scaling = ((width / model_bbox.width) + (height / model_bbox.height)) / 2; cv::Vec3f rotation_init(rotation[0], rotation[1], rotation[2]); - cv::Matx33f R = Euler2RotationMatrix(rotation_init); + cv::Matx33f R = Utilities::Euler2RotationMatrix(rotation_init); cv::Vec2f translation((min_x + max_x) / 2.0, (min_y + max_y) / 2.0); cv::Mat_ loc_params(this->NumberOfModes(),1, 0.0); @@ -553,7 +554,7 @@ void PDM::CalcParams(cv::Vec6f& out_params_global, cv::Mat_& out_params_l translation[0] = glob_params[4]; translation[1] = glob_params[5]; - R = Euler2RotationMatrix(rotation_init); + R = Utilities::Euler2RotationMatrix(rotation_init); R_2D(0,0) = R(0,0);R_2D(0,1) = R(0,1); R_2D(0,2) = R(0,2); R_2D(1,0) = R(1,0);R_2D(1,1) = R(1,1); R_2D(1,2) = R(1,2); diff --git a/lib/local/GazeAnalyser/CMakeLists.txt b/lib/local/GazeAnalyser/CMakeLists.txt index da01951..b62d286 100644 --- a/lib/local/GazeAnalyser/CMakeLists.txt +++ b/lib/local/GazeAnalyser/CMakeLists.txt @@ -10,6 +10,9 @@ include_directories(../../3rdParty/OpenBLAS/include) #LandmarkDetector library include_directories(../../local/LandmarkDetector/include) +#Utilities library +include_directories(../../local/Utilities/include) + SET(SOURCE src/GazeEstimation.cpp ) diff --git a/lib/local/GazeAnalyser/GazeAnalyser.vcxproj b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj index 9b22cad..844f347 100644 --- a/lib/local/GazeAnalyser/GazeAnalyser.vcxproj +++ b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj @@ -91,7 +91,7 @@ Level3 Disabled true - ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 @@ -100,7 +100,7 @@ Level3 Disabled true - ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN64;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) AdvancedVectorExtensions @@ -113,7 +113,7 @@ true - ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) true WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) StreamingSIMDExtensions2 @@ -126,15 +126,17 @@ Level3 - MaxSpeed + Full true - true + false - ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + ./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) true WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) AdvancedVectorExtensions + AnySuitable + Speed true diff --git a/lib/local/GazeAnalyser/include/GazeEstimation.h b/lib/local/GazeAnalyser/include/GazeEstimation.h index ce6f0af..c7f1b3a 100644 --- a/lib/local/GazeAnalyser/include/GazeEstimation.h +++ b/lib/local/GazeAnalyser/include/GazeEstimation.h @@ -43,7 +43,6 @@ namespace GazeAnalysis { void EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye); - void DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy); // Getting the gaze angle in radians with respect to the world coordinates (camera plane), when looking ahead straight at camera plane the gaze angle will be (0,0) cv::Vec2d GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2); diff --git a/lib/local/GazeAnalyser/src/GazeEstimation.cpp b/lib/local/GazeAnalyser/src/GazeEstimation.cpp index d9456b8..17302d3 100644 --- a/lib/local/GazeAnalyser/src/GazeEstimation.cpp +++ b/lib/local/GazeAnalyser/src/GazeEstimation.cpp @@ -42,15 +42,12 @@ #include "LandmarkDetectorUtils.h" #include "LandmarkDetectorFunc.h" +#include "RotationHelpers.h" using namespace std; using namespace GazeAnalysis; -// For subpixel accuracy drawing -const int gaze_draw_shiftbits = 4; -const int gaze_draw_multiplier = 1 << 4; - cv::Point3f RaySphereIntersect(cv::Point3f rayOrigin, cv::Point3f rayDir, cv::Point3f sphereOrigin, float sphereRadius){ float dx = rayDir.x; @@ -93,7 +90,7 @@ void GazeAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po { cv::Vec6d headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); cv::Vec3d eulerAngles(headPose(3), headPose(4), headPose(5)); - cv::Matx33d rotMat = LandmarkDetector::Euler2RotationMatrix(eulerAngles); + cv::Matx33d rotMat = Utilities::Euler2RotationMatrix(eulerAngles); int part = -1; for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) @@ -148,48 +145,4 @@ cv::Vec2d GazeAnalysis::GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& ga return cv::Vec2d(x_angle, y_angle); -} -void GazeAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy) -{ - - cv::Mat cameraMat = (cv::Mat_(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0); - - int part_left = -1; - int part_right = -1; - for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) - { - if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0) - { - part_left = i; - } - if (clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0) - { - part_right = i; - } - } - - cv::Mat eyeLdmks3d_left = clnf_model.hierarchical_models[part_left].GetShape(fx, fy, cx, cy); - cv::Point3f pupil_left = GetPupilPosition(eyeLdmks3d_left); - - cv::Mat eyeLdmks3d_right = clnf_model.hierarchical_models[part_right].GetShape(fx, fy, cx, cy); - cv::Point3f pupil_right = GetPupilPosition(eyeLdmks3d_right); - - vector points_left; - points_left.push_back(cv::Point3d(pupil_left)); - points_left.push_back(cv::Point3d(pupil_left + gazeVecAxisLeft*50.0)); - - vector points_right; - points_right.push_back(cv::Point3d(pupil_right)); - points_right.push_back(cv::Point3d(pupil_right + gazeVecAxisRight*50.0)); - - cv::Mat_ proj_points; - cv::Mat_ mesh_0 = (cv::Mat_(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z); - LandmarkDetector::Project(proj_points, mesh_0, fx, fy, cx, cy); - cv::line(img, cv::Point(cvRound(proj_points.at(0,0) * (double)gaze_draw_multiplier), cvRound(proj_points.at(0, 1) * (double)gaze_draw_multiplier)), - cv::Point(cvRound(proj_points.at(1, 0) * (double)gaze_draw_multiplier), cvRound(proj_points.at(1, 1) * (double)gaze_draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, gaze_draw_shiftbits); - - cv::Mat_ mesh_1 = (cv::Mat_(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z); - LandmarkDetector::Project(proj_points, mesh_1, fx, fy, cx, cy); - cv::line(img, cv::Point(cvRound(proj_points.at(0, 0) * (double)gaze_draw_multiplier), cvRound(proj_points.at(0, 1) * (double)gaze_draw_multiplier)), - cv::Point(cvRound(proj_points.at(1, 0) * (double)gaze_draw_multiplier), cvRound(proj_points.at(1, 1) * (double)gaze_draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, gaze_draw_shiftbits); } \ No newline at end of file diff --git a/lib/local/LandmarkDetector/CMakeLists.txt b/lib/local/LandmarkDetector/CMakeLists.txt index 1ff9878..fe5329e 100644 --- a/lib/local/LandmarkDetector/CMakeLists.txt +++ b/lib/local/LandmarkDetector/CMakeLists.txt @@ -6,6 +6,9 @@ include_directories(${BOOST_INCLUDE_DIR}) #OpenBlas library include_directories(../../3rdParty/OpenBLAS/include) +#Utilities library +include_directories(../../local/Utilities/include) + SET(SOURCE src/CCNF_patch_expert.cpp src/LandmarkDetectionValidator.cpp diff --git a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj index f2d5744..dc9bfe3 100644 --- a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj +++ b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj @@ -93,7 +93,7 @@ Disabled - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) true EnableFastChecks @@ -117,7 +117,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c Disabled - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN64;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebugDLL @@ -141,7 +141,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c Full true - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) MultiThreadedDLL false @@ -165,8 +165,8 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c Full - true - ./include;%(AdditionalIncludeDirectories) + false + ./include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) MultiThreadedDLL false @@ -178,6 +178,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c false /Zm300 %(AdditionalOptions) true + AnySuitable $(OutDir)$(TargetName)$(TargetExt) diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h index 91f997e..04a1550 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h @@ -93,16 +93,13 @@ public: dlib::frontal_face_detector face_detector_HOG; - // Validate if the detected landmarks are correct using an SVR regressor + // Validate if the detected landmarks are correct using a predictor on detected landmarks DetectionValidator landmark_validator; - // Indicating if landmark detection succeeded (based on SVR validator) + // Indicating if landmark detection succeeded (based on detection validator) bool detection_success; - // Indicating if the tracking has been initialised (for video based tracking) - bool tracking_initialised; - - // The actual output of the regressor (-1 is perfect detection 1 is worst detection) + // Representing how confident we are that tracking succeeds (0 - complete failure, 1 - perfect success) double detection_certainty; // Indicator if eye model is there for eye detection @@ -130,11 +127,7 @@ public: // Useful when resetting or initialising the model closer to a specific location (when multiple faces are present) cv::Point_ preference_det; - - // Useful to control where face detections should not occur (for live demos etc.) - double detect_Z_max = -1; // Do not detect faces further than this in mm. (-1) refers to detecting all faces - cv::Rect_ detect_ROI = cv::Rect(0, 0, 1, 1); // the face detection bounds (0,0,1,1) means full image (0.25,0.25,0.5,0.5) would imply region of the face only - + // A default constructor CLNF(); @@ -178,8 +171,17 @@ public: // Helper reading function void Read_CLNF(string clnf_location); + // Allows to set initialization accross hierarchical models as well + bool IsInitialized() const { return tracking_initialised; } + void SetInitialized(bool initialized); + void SetDetectionSuccess(bool detection_success); + private: + + // Indicating if the tracking has been initialised (for video based tracking) + bool tracking_initialised; + // the speedup of RLMS using precalculated KDE responses (described in Saragih 2011 RLMS paper) map > kde_resp_precalc; diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h b/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h index 77ed168..f8840d0 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h @@ -102,9 +102,6 @@ struct FaceModelParameters // Should the parameters be refined for different scales bool refine_parameters; - // Using the brand new and experimental gaze tracker - bool track_gaze; - FaceModelParameters(); FaceModelParameters(vector &arguments); diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index 797e5c6..a716dda 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -48,18 +48,6 @@ namespace LandmarkDetector //=========================================================================== // Defining a set of useful utility functions to be used within CLNF - - //============================================================================================= - // Helper functions for parsing the inputs - //============================================================================================= - void get_video_input_output_params(vector &input_video_file, vector &output_files, - vector &output_video_files, string &output_codec, vector &arguments); - - void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector &arguments); - - void get_image_input_output_params(vector &input_image_files, vector &output_feature_files, vector &output_pose_files, vector &output_image_files, - vector> &input_bounding_boxes, vector &arguments); - //=========================================================================== // Fast patch expert response computation (linear model across a ROI) using normalised cross-correlation //=========================================================================== @@ -82,13 +70,6 @@ namespace LandmarkDetector //=========================================================================== // Visualisation functions //=========================================================================== - void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy); - void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy); - - // Drawing face bounding box - vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); - void DrawBox(vector> lines, cv::Mat image, cv::Scalar color, int thickness); - vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities); vector CalculateVisibleLandmarks(const CLNF& clnf_model); vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model); @@ -96,45 +77,23 @@ namespace LandmarkDetector vector CalculateAllLandmarks(const cv::Mat_& shape2D); vector CalculateAllLandmarks(const CLNF& clnf_model); vector CalculateAllEyeLandmarks(const CLNF& clnf_model); - - void DrawLandmarks(cv::Mat img, vector landmarks); - - void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities); - void Draw(cv::Mat img, const cv::Mat_& shape2D); - void Draw(cv::Mat img, const CLNF& clnf_model); - - - //=========================================================================== - // Angle representation conversion helpers - //=========================================================================== - cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles); - - // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign - cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix); - - cv::Vec3d Euler2AxisAngle(const cv::Vec3d& euler); - - cv::Vec3d AxisAngle2Euler(const cv::Vec3d& axis_angle); - - cv::Matx33d AxisAngle2RotationMatrix(const cv::Vec3d& axis_angle); - - cv::Vec3d RotationMatrix2AxisAngle(const cv::Matx33d& rotation_matrix); + vector Calculate3DEyeLandmarks(const CLNF& clnf_model, double fx, double fy, double cx, double cy); //============================================================================ // Face detection helpers //============================================================================ // Face detection using Haar cascade classifier - bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); - bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, cv::CascadeClassifier& classifier, double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); + bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity); + bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, cv::CascadeClassifier& classifier); // The preference point allows for disambiguation if multiple faces are present (pick the closest one), if it is not set the biggest face is chosen - bool DetectSingleFace(cv::Rect_& o_region, const cv::Mat_& intensity, cv::CascadeClassifier& classifier, const cv::Point preference = cv::Point(-1,-1), double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); + bool DetectSingleFace(cv::Rect_& o_region, const cv::Mat_& intensity, cv::CascadeClassifier& classifier, const cv::Point preference = cv::Point(-1,-1)); // Face detection using HOG-SVM classifier - bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, std::vector& confidences, double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); - bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, dlib::frontal_face_detector& classifier, std::vector& confidences, double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); + bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, std::vector& confidences); + bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, dlib::frontal_face_detector& classifier, std::vector& confidences); // The preference point allows for disambiguation if multiple faces are present (pick the closest one), if it is not set the biggest face is chosen - bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& intensity, dlib::frontal_face_detector& classifier, double& confidence, const cv::Point preference = cv::Point(-1,-1), double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0,0.0,1.0,1.0)); + bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& intensity, dlib::frontal_face_detector& classifier, double& confidence, const cv::Point preference = cv::Point(-1,-1)); //============================================================================ // Matrix reading functionality diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp index 2a3b90e..23d4698 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp @@ -488,6 +488,10 @@ double DetectionValidator::Check(const cv::Vec3d& orientation, const cv::Mat_ +#include "RotationHelpers.h" // OpenCV includes #include @@ -83,7 +84,7 @@ cv::Vec6d LandmarkDetector::GetPose(const CLNF& clnf_model, float fx, float fy, cv::solvePnP(landmarks_3D, landmarks_2D, camera_matrix, cv::Mat(), vec_rot, vec_trans, true); - cv::Vec3d euler = LandmarkDetector::AxisAngle2Euler(vec_rot); + cv::Vec3d euler = Utilities::AxisAngle2Euler(vec_rot); return cv::Vec6d(vec_trans[0], vec_trans[1], vec_trans[2], euler[0], euler[1], euler[2]); } @@ -136,12 +137,12 @@ cv::Vec6d LandmarkDetector::GetPoseWRTCamera(const CLNF& clnf_model, float fx, f double z_y = cv::sqrt(vec_trans[1] * vec_trans[1] + vec_trans[2] * vec_trans[2]); double eul_y = -atan2(vec_trans[0], z_y); - cv::Matx33d camera_rotation = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(eul_x, eul_y, 0)); - cv::Matx33d head_rotation = LandmarkDetector::AxisAngle2RotationMatrix(vec_rot); + cv::Matx33d camera_rotation = Utilities::Euler2RotationMatrix(cv::Vec3d(eul_x, eul_y, 0)); + cv::Matx33d head_rotation = Utilities::AxisAngle2RotationMatrix(vec_rot); cv::Matx33d corrected_rotation = camera_rotation * head_rotation; - cv::Vec3d euler_corrected = LandmarkDetector::RotationMatrix2Euler(corrected_rotation); + cv::Vec3d euler_corrected = Utilities::RotationMatrix2Euler(corrected_rotation); return cv::Vec6d(vec_trans[0], vec_trans[1], vec_trans[2], euler_corrected[0], euler_corrected[1], euler_corrected[2]); } @@ -214,10 +215,10 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i // and using a smaller search area // Indicating that this is a first detection in video sequence or after restart - bool initial_detection = !clnf_model.tracking_initialised; + bool initial_detection = !clnf_model.IsInitialized(); // Only do it if there was a face detection at all - if(clnf_model.tracking_initialised) + if(clnf_model.IsInitialized()) { // The area of interest search size will depend if the previous track was successful @@ -253,8 +254,8 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i // This is used for both detection (if it the tracking has not been initialised yet) or if the tracking failed (however we do this every n frames, for speed) // This also has the effect of an attempt to reinitialise just after the tracking has failed, which is useful during large motions - if((!clnf_model.tracking_initialised && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0) - || (clnf_model.tracking_initialised && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0)) + if((!clnf_model.IsInitialized() && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0) + || (clnf_model.IsInitialized() && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0)) { cv::Rect_ bounding_box; @@ -276,33 +277,21 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i bool face_detection_success; - // For pruning face detections - double min_size; - if (clnf_model.detect_Z_max == -1) - { - min_size = -1; - } - else - { - double fx_est = 500.0 * (grayscale_image.cols / 640.0); - min_size = fx_est * 150.0 / clnf_model.detect_Z_max; - } - if(params.curr_face_detector == FaceModelParameters::HOG_SVM_DETECTOR) { double confidence; - face_detection_success = LandmarkDetector::DetectSingleFaceHOG(bounding_box, grayscale_image, clnf_model.face_detector_HOG, confidence, preference_det, min_size, clnf_model.detect_ROI); + face_detection_success = LandmarkDetector::DetectSingleFaceHOG(bounding_box, grayscale_image, clnf_model.face_detector_HOG, confidence, preference_det); } else if(params.curr_face_detector == FaceModelParameters::HAAR_DETECTOR) { - face_detection_success = LandmarkDetector::DetectSingleFace(bounding_box, grayscale_image, clnf_model.face_detector_HAAR, preference_det, min_size, clnf_model.detect_ROI); + face_detection_success = LandmarkDetector::DetectSingleFace(bounding_box, grayscale_image, clnf_model.face_detector_HAAR, preference_det); } // Attempt to detect landmarks using the detected face (if unseccessful the detection will be ignored) if(face_detection_success) { // Indicate that tracking has started as a face was detected - clnf_model.tracking_initialised = true; + clnf_model.SetInitialized(true); // Keep track of old model values so that they can be restored if redetection fails cv::Vec6d params_global_init = clnf_model.params_global; @@ -347,7 +336,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i } // if the model has not been initialised yet class it as a failure - if(!clnf_model.tracking_initialised) + if(!clnf_model.IsInitialized()) { clnf_model.failures_in_a_row++; } @@ -355,7 +344,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i // un-initialise the tracking if( clnf_model.failures_in_a_row > 100) { - clnf_model.tracking_initialised = false; + clnf_model.SetInitialized(false); } return clnf_model.detection_success; @@ -371,7 +360,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i clnf_model.pdm.CalcParams(clnf_model.params_global, bounding_box, clnf_model.params_local); // indicate that face was detected so initialisation is not necessary - clnf_model.tracking_initialised = true; + clnf_model.SetInitialized(true); } return DetectLandmarksInVideo(grayscale_image, clnf_model, params); @@ -475,6 +464,9 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_ &grayscale_i clnf_model.hierarchical_models[part].landmark_likelihoods = best_landmark_likelihoods_h[part].clone(); } + // To indicate that tracking/detection started and the values are valid, we assume that there is a face in the bounding box + clnf_model.SetInitialized(true); + return best_success; } diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index 5b5aa1a..5e3fecf 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -45,6 +45,7 @@ // Local includes #include +#include using namespace LandmarkDetector; @@ -66,10 +67,10 @@ CLNF::CLNF(string fname) } // Copy constructor (makes a deep copy of CLNF) -CLNF::CLNF(const CLNF& other): pdm(other.pdm), params_local(other.params_local.clone()), params_global(other.params_global), detected_landmarks(other.detected_landmarks.clone()), - landmark_likelihoods(other.landmark_likelihoods.clone()), patch_experts(other.patch_experts), landmark_validator(other.landmark_validator), face_detector_location(other.face_detector_location), - hierarchical_mapping(other.hierarchical_mapping), hierarchical_models(other.hierarchical_models), hierarchical_model_names(other.hierarchical_model_names), - hierarchical_params(other.hierarchical_params), eye_model(other.eye_model), detect_Z_max(other.detect_Z_max), detect_ROI(other.detect_ROI) +CLNF::CLNF(const CLNF& other) : pdm(other.pdm), params_local(other.params_local.clone()), params_global(other.params_global), detected_landmarks(other.detected_landmarks.clone()), +landmark_likelihoods(other.landmark_likelihoods.clone()), patch_experts(other.patch_experts), landmark_validator(other.landmark_validator), face_detector_location(other.face_detector_location), +hierarchical_mapping(other.hierarchical_mapping), hierarchical_models(other.hierarchical_models), hierarchical_model_names(other.hierarchical_model_names), +hierarchical_params(other.hierarchical_params), eye_model(other.eye_model) { this->detection_success = other.detection_success; this->tracking_initialised = other.tracking_initialised; @@ -78,20 +79,20 @@ CLNF::CLNF(const CLNF& other): pdm(other.pdm), params_local(other.params_local.c this->failures_in_a_row = other.failures_in_a_row; // Load the CascadeClassifier (as it does not have a proper copy constructor) - if(!face_detector_location.empty()) + if (!face_detector_location.empty()) { this->face_detector_HAAR.load(face_detector_location); } // Make sure the matrices are allocated properly this->triangulations.resize(other.triangulations.size()); - for(size_t i = 0; i < other.triangulations.size(); ++i) + for (size_t i = 0; i < other.triangulations.size(); ++i) { // Make sure the matrix is copied. this->triangulations[i] = other.triangulations[i].clone(); } // Make sure the matrices are allocated properly - for(std::map>::const_iterator it = other.kde_resp_precalc.begin(); it!= other.kde_resp_precalc.end(); it++) + for (std::map>::const_iterator it = other.kde_resp_precalc.begin(); it != other.kde_resp_precalc.end(); it++) { // Make sure the matrix is copied. this->kde_resp_precalc.insert(std::pair>(it->first, it->second.clone())); @@ -110,8 +111,8 @@ CLNF & CLNF::operator= (const CLNF& other) params_local = other.params_local.clone(); params_global = other.params_global; detected_landmarks = other.detected_landmarks.clone(); - - landmark_likelihoods =other.landmark_likelihoods.clone(); + + landmark_likelihoods = other.landmark_likelihoods.clone(); patch_experts = Patch_experts(other.patch_experts); landmark_validator = DetectionValidator(other.landmark_validator); face_detector_location = other.face_detector_location; @@ -125,20 +126,20 @@ CLNF & CLNF::operator= (const CLNF& other) this->eye_model = other.eye_model; // Load the CascadeClassifier (as it does not have a proper copy constructor) - if(!face_detector_location.empty()) + if (!face_detector_location.empty()) { this->face_detector_HAAR.load(face_detector_location); } // Make sure the matrices are allocated properly this->triangulations.resize(other.triangulations.size()); - for(size_t i = 0; i < other.triangulations.size(); ++i) + for (size_t i = 0; i < other.triangulations.size(); ++i) { // Make sure the matrix is copied. this->triangulations[i] = other.triangulations[i].clone(); } // Make sure the matrices are allocated properly - for(std::map>::const_iterator it = other.kde_resp_precalc.begin(); it!= other.kde_resp_precalc.end(); it++) + for (std::map>::const_iterator it = other.kde_resp_precalc.begin(); it != other.kde_resp_precalc.end(); it++) { // Make sure the matrix is copied. this->kde_resp_precalc.insert(std::pair>(it->first, it->second.clone())); @@ -149,9 +150,6 @@ CLNF & CLNF::operator= (const CLNF& other) this->hierarchical_models = other.hierarchical_models; this->hierarchical_model_names = other.hierarchical_model_names; this->hierarchical_params = other.hierarchical_params; - - this->detect_Z_max = other.detect_Z_max; - this->detect_ROI = detect_ROI; } face_detector_HOG = dlib::get_frontal_face_detector(); @@ -192,8 +190,6 @@ CLNF::CLNF(const CLNF&& other) this->eye_model = other.eye_model; - this->detect_Z_max = other.detect_Z_max; - this->detect_ROI = detect_ROI; } // Assignment operator for rvalues @@ -228,9 +224,6 @@ CLNF & CLNF::operator= (const CLNF&& other) this->hierarchical_params = other.hierarchical_params; this->eye_model = other.eye_model; - - this->detect_Z_max = other.detect_Z_max; - this->detect_ROI = detect_ROI; return *this; } @@ -241,7 +234,7 @@ void CLNF::Read_CLNF(string clnf_location) // Location of modules ifstream locations(clnf_location.c_str(), ios_base::in); - if(!locations.is_open()) + if (!locations.is_open()) { cout << "Couldn't open the CLNF model file aborting" << endl; cout.flush(); @@ -249,7 +242,7 @@ void CLNF::Read_CLNF(string clnf_location) } string line; - + vector intensity_expert_locations; vector ccnf_expert_locations; @@ -258,8 +251,8 @@ void CLNF::Read_CLNF(string clnf_location) // The main file contains the references to other files while (!locations.eof()) - { - + { + getline(locations, line); stringstream lineStream(line); @@ -269,30 +262,30 @@ void CLNF::Read_CLNF(string clnf_location) // figure out which module is to be read from which file lineStream >> module; - + getline(lineStream, location); - if(location.size() > 0) + if (location.size() > 0) location.erase(location.begin()); // remove the first space - - // remove carriage return at the end for compatibility with unix systems - if(location.size() > 0 && location.at(location.size()-1) == '\r') + + // remove carriage return at the end for compatibility with unix systems + if (location.size() > 0 && location.at(location.size() - 1) == '\r') { - location = location.substr(0, location.size()-1); + location = location.substr(0, location.size() - 1); } // append the lovstion to root location (boost syntax) location = (root / location).string(); - - if (module.compare("PDM") == 0) - { + + if (module.compare("PDM") == 0) + { cout << "Reading the PDM module from: " << location << "...."; pdm.Read(location); cout << "Done" << endl; } - else if (module.compare("Triangulations") == 0) - { + else if (module.compare("Triangulations") == 0) + { cout << "Reading the Triangulations module from: " << location << "...."; ifstream triangulationFile(location.c_str(), ios_base::in); @@ -304,23 +297,23 @@ void CLNF::Read_CLNF(string clnf_location) // read in the triangulations triangulations.resize(numViews); - for(int i = 0; i < numViews; ++i) + for (int i = 0; i < numViews; ++i) { LandmarkDetector::SkipComments(triangulationFile); LandmarkDetector::ReadMat(triangulationFile, triangulations[i]); } cout << "Done" << endl; } - else if(module.compare("PatchesIntensity") == 0) + else if (module.compare("PatchesIntensity") == 0) { intensity_expert_locations.push_back(location); } - else if(module.compare("PatchesCCNF") == 0) + else if (module.compare("PatchesCCNF") == 0) { ccnf_expert_locations.push_back(location); } } - + // Initialise the patch experts patch_experts.Read(intensity_expert_locations, ccnf_expert_locations); @@ -333,28 +326,24 @@ void CLNF::Read(string main_location) { cout << "Reading the CLNF landmark detector/tracker from: " << main_location << endl; - + ifstream locations(main_location.c_str(), ios_base::in); - if(!locations.is_open()) + if (!locations.is_open()) { cout << "Couldn't open the model file, aborting" << endl; return; } string line; - + // The other module locations should be defined as relative paths from the main model - boost::filesystem::path root = boost::filesystem::path(main_location).parent_path(); - - // Initializing defaults that could be overwriten - detect_Z_max = -1; - detect_ROI = cv::Rect_(0, 0, 1, 1); + boost::filesystem::path root = boost::filesystem::path(main_location).parent_path(); // Assume no eye model, unless read-in eye_model = false; // The main file contains the references to other files while (!locations.eof()) - { + { getline(locations, line); stringstream lineStream(line); @@ -364,42 +353,42 @@ void CLNF::Read(string main_location) // figure out which module is to be read from which file lineStream >> module; - + lineStream >> location; - + // remove carriage return at the end for compatibility with unix systems - if(location.size() > 0 && location.at(location.size()-1) == '\r') + if (location.size() > 0 && location.at(location.size() - 1) == '\r') { - location = location.substr(0, location.size()-1); + location = location.substr(0, location.size() - 1); } // append to root location = (root / location).string(); - if (module.compare("LandmarkDetector") == 0) - { + if (module.compare("LandmarkDetector") == 0) + { cout << "Reading the landmark detector module from: " << location << endl; // The CLNF module includes the PDM and the patch experts Read_CLNF(location); } - else if(module.compare("LandmarkDetector_part") == 0) + else if (module.compare("LandmarkDetector_part") == 0) { string part_name; lineStream >> part_name; cout << "Reading part based module...." << part_name << endl; vector> mappings; - while(!lineStream.eof()) + while (!lineStream.eof()) { int ind_in_main; lineStream >> ind_in_main; - + int ind_in_part; lineStream >> ind_in_part; mappings.push_back(pair(ind_in_main, ind_in_part)); } - + this->hierarchical_mapping.push_back(mappings); CLNF part_model(location); @@ -413,9 +402,9 @@ void CLNF::Read(string main_location) params.refine_hierarchical = false; params.refine_parameters = false; - if(part_name.compare("left_eye") == 0 || part_name.compare("right_eye") == 0) + if (part_name.compare("left_eye") == 0 || part_name.compare("right_eye") == 0) { - + vector windows_large; windows_large.push_back(5); windows_large.push_back(3); @@ -431,7 +420,7 @@ void CLNF::Read(string main_location) params.reg_factor = 0.1; params.sigma = 2; } - else if(part_name.compare("left_eye_28") == 0 || part_name.compare("right_eye_28") == 0) + else if (part_name.compare("left_eye_28") == 0 || part_name.compare("right_eye_28") == 0) { vector windows_large; windows_large.push_back(3); @@ -453,7 +442,7 @@ void CLNF::Read(string main_location) eye_model = true; } - else if(part_name.compare("mouth") == 0) + else if (part_name.compare("mouth") == 0) { vector windows_large; windows_large.push_back(7); @@ -470,7 +459,7 @@ void CLNF::Read(string main_location) params.reg_factor = 1.0; params.sigma = 2.0; } - else if(part_name.compare("brow") == 0) + else if (part_name.compare("brow") == 0) { vector windows_large; windows_large.push_back(11); @@ -487,7 +476,7 @@ void CLNF::Read(string main_location) params.reg_factor = 10.0; params.sigma = 3.5; } - else if(part_name.compare("inner") == 0) + else if (part_name.compare("inner") == 0) { vector windows_large; windows_large.push_back(9); @@ -509,45 +498,53 @@ void CLNF::Read(string main_location) cout << "Done" << endl; } else if (module.compare("DetectionValidator") == 0) - { + { cout << "Reading the landmark validation module...."; landmark_validator.Read(location); cout << "Done" << endl; } - else if (module.compare("DetectorConstaints") == 0) - { - cout << "Reading detectir constraints..."; - lineStream >> detect_Z_max; - lineStream >> detect_ROI.x; - lineStream >> detect_ROI.y; - lineStream >> detect_ROI.width; - lineStream >> detect_ROI.height; - cout << "Done" << endl; - } } - + detected_landmarks.create(2 * pdm.NumberOfPoints(), 1); detected_landmarks.setTo(0); - detection_success = false; - tracking_initialised = false; + SetDetectionSuccess(false); + SetInitialized(false); model_likelihood = -10; // very low - detection_certainty = 1; // very uncertain + detection_certainty = 0; // very uncertain - // Initialising default values for the rest of the variables + // Initialising default values for the rest of the variables - // local parameters (shape) + // local parameters (shape) params_local.create(pdm.NumberOfModes(), 1); params_local.setTo(0.0); // global parameters (pose) [scale, euler_x, euler_y, euler_z, tx, ty] - params_global = cv::Vec6d(1, 0, 0, 0, 0, 0); + params_global = cv::Vec6d(0, 0, 0, 0, 0, 0); failures_in_a_row = -1; - preference_det.x = -1; - preference_det.y = -1; +} +void CLNF::SetDetectionSuccess(bool success) +{ + this->detection_success = success; + + for (size_t i = 0; i < hierarchical_models.size(); ++i) + { + hierarchical_models[i].SetDetectionSuccess(success); + } + +} + +void CLNF::SetInitialized(bool initialized) +{ + this->tracking_initialised = initialized; + + for (size_t i = 0; i < hierarchical_models.size(); ++i) + { + hierarchical_models[i].SetInitialized(initialized); + } } @@ -556,12 +553,12 @@ void CLNF::Reset() { detected_landmarks.setTo(0); - detection_success = false; - tracking_initialised = false; + SetDetectionSuccess(false); + SetInitialized(false); model_likelihood = -10; // very low - detection_certainty = 1; // very uncertain + detection_certainty = 0; // very uncertain - // local parameters (shape) + // local parameters (shape) params_local.setTo(0.0); // global parameters (pose) [scale, euler_x, euler_y, euler_z, tx, ty] @@ -592,19 +589,14 @@ bool CLNF::DetectLandmarks(const cv::Mat_ &image, FaceModelParameters& pa bool fit_success = Fit(image, params.window_sizes_current, params); // Store the landmarks converged on in detected_landmarks - pdm.CalcShape2D(detected_landmarks, params_local, params_global); - - if(params.refine_hierarchical && hierarchical_models.size() > 0) + pdm.CalcShape2D(detected_landmarks, params_local, params_global); + + if (params.refine_hierarchical && hierarchical_models.size() > 0) { - bool parts_used = false; + bool parts_used = false; // Do the hierarchical models in parallel - tbb::parallel_for(0, (int)hierarchical_models.size(), [&](int part_model){ - { - // Only do the synthetic eye models if we're doing gaze - if (!((hierarchical_model_names[part_model].compare("right_eye_28") == 0 || - hierarchical_model_names[part_model].compare("left_eye_28") == 0) - && !params.track_gaze)) + tbb::parallel_for(0, (int)hierarchical_models.size(), [&](int part_model) { { int n_part_points = hierarchical_models[part_model].pdm.NumberOfPoints(); @@ -639,55 +631,49 @@ bool CLNF::DetectLandmarks(const cv::Mat_ &image, FaceModelParameters& pa hierarchical_models[part_model].pdm.CalcShape2D(hierarchical_models[part_model].detected_landmarks, hierarchical_models[part_model].params_local, hierarchical_models[part_model].params_global); } } - } }); // Recompute main model based on the fit part models - if(parts_used) + if (parts_used) { for (size_t part_model = 0; part_model < hierarchical_models.size(); ++part_model) { vector> mappings = this->hierarchical_mapping[part_model]; - if (!((hierarchical_model_names[part_model].compare("right_eye_28") == 0 || - hierarchical_model_names[part_model].compare("left_eye_28") == 0) - && !params.track_gaze)) + // Reincorporate the models into main tracker + for (size_t mapping_ind = 0; mapping_ind < mappings.size(); ++mapping_ind) { - // Reincorporate the models into main tracker - for (size_t mapping_ind = 0; mapping_ind < mappings.size(); ++mapping_ind) - { - detected_landmarks.at(mappings[mapping_ind].first) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second); - detected_landmarks.at(mappings[mapping_ind].first + pdm.NumberOfPoints()) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second + hierarchical_models[part_model].pdm.NumberOfPoints()); - } + detected_landmarks.at(mappings[mapping_ind].first) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second); + detected_landmarks.at(mappings[mapping_ind].first + pdm.NumberOfPoints()) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second + hierarchical_models[part_model].pdm.NumberOfPoints()); } } - pdm.CalcParams(params_global, params_local, detected_landmarks); + pdm.CalcParams(params_global, params_local, detected_landmarks); pdm.CalcShape2D(detected_landmarks, params_local, params_global); } } // Check detection correctness - if(params.validate_detections && fit_success) + if (params.validate_detections && fit_success) { cv::Vec3d orientation(params_global[1], params_global[2], params_global[3]); detection_certainty = landmark_validator.Check(orientation, image, detected_landmarks); - detection_success = detection_certainty < params.validation_boundary; + detection_success = detection_certainty > params.validation_boundary; } else { detection_success = fit_success; - if(fit_success) + if (fit_success) { - detection_certainty = -1; + detection_certainty = 1; } else { - detection_certainty = 1; + detection_certainty = 0; } } @@ -699,13 +685,13 @@ bool CLNF::DetectLandmarks(const cv::Mat_ &image, FaceModelParameters& pa bool CLNF::Fit(const cv::Mat_& im, const std::vector& window_sizes, const FaceModelParameters& parameters) { // Making sure it is a single channel image - assert(im.channels() == 1); - - // Placeholder for the landmarks - cv::Mat_ current_shape(2 * pdm.NumberOfPoints() , 1, 0.0); + assert(im.channels() == 1); + + // Placeholder for the landmarks + cv::Mat_ current_shape(2 * pdm.NumberOfPoints(), 1, 0.0); + + int n = pdm.NumberOfPoints(); - int n = pdm.NumberOfPoints(); - int num_scales = patch_experts.patch_scaling.size(); // Storing the patch expert response maps @@ -718,27 +704,27 @@ bool CLNF::Fit(const cv::Mat_& im, const std::vector& window_sizes, FaceModelParameters tmp_parameters = parameters; // Optimise the model across a number of areas of interest (usually in descending window size and ascending scale size) - for(int scale = 0; scale < num_scales; scale++) + for (int scale = 0; scale < num_scales; scale++) { int window_size = window_sizes[scale]; - if(window_size == 0 || 0.9 * patch_experts.patch_scaling[scale] > params_global[0]) + if (window_size == 0 || 0.9 * patch_experts.patch_scaling[scale] > params_global[0]) continue; // The patch expert response computation - patch_experts.Response(patch_expert_responses, sim_ref_to_img, sim_img_to_ref, im, pdm, params_global, params_local, window_size, scale); - - if(parameters.refine_parameters == true) + patch_experts.Response(patch_expert_responses, sim_ref_to_img, sim_img_to_ref, im, pdm, params_global, params_local, window_size, scale); + + if (parameters.refine_parameters == true) { // Adapt the parameters based on scale (wan't to reduce regularisation as scale increases, but increa sigma and tikhonov) - tmp_parameters.reg_factor = parameters.reg_factor - 15 * log(patch_experts.patch_scaling[scale]/0.25)/log(2); - - if(tmp_parameters.reg_factor <= 0) + tmp_parameters.reg_factor = parameters.reg_factor - 15 * log(patch_experts.patch_scaling[scale] / 0.25) / log(2); + + if (tmp_parameters.reg_factor <= 0) tmp_parameters.reg_factor = 0.001; - tmp_parameters.sigma = parameters.sigma + 0.25 * log(patch_experts.patch_scaling[scale]/0.25)/log(2); - tmp_parameters.weight_factor = parameters.weight_factor + 2 * parameters.weight_factor * log(patch_experts.patch_scaling[scale]/0.25)/log(2); + tmp_parameters.sigma = parameters.sigma + 0.25 * log(patch_experts.patch_scaling[scale] / 0.25) / log(2); + tmp_parameters.weight_factor = parameters.weight_factor + 2 * parameters.weight_factor * log(patch_experts.patch_scaling[scale] / 0.25) / log(2); } // Get the current landmark locations @@ -752,9 +738,9 @@ bool CLNF::Fit(const cv::Mat_& im, const std::vector& window_sizes, // non-rigid optimisation this->model_likelihood = this->NU_RLMS(params_global, params_local, patch_expert_responses, cv::Vec6d(params_global), params_local.clone(), current_shape, sim_img_to_ref, sim_ref_to_img, window_size, view_id, false, scale, this->landmark_likelihoods, tmp_parameters); - + // Can't track very small images reliably (less than ~30px across) - if(params_global[0] < 0.25) + if (params_global[0] < 0.25) { cout << "Face too small for landmark detection" << endl; return false; @@ -766,38 +752,38 @@ bool CLNF::Fit(const cv::Mat_& im, const std::vector& window_sizes, void CLNF::NonVectorisedMeanShift_precalc_kde(cv::Mat_& out_mean_shifts, const vector >& patch_expert_responses, const cv::Mat_ &dxs, const cv::Mat_ &dys, int resp_size, float a, int scale, int view_id, map >& kde_resp_precalc) { - + int n = dxs.rows; - + cv::Mat_ kde_resp; float step_size = 0.1; // if this has not been precomputer, precompute it, otherwise use it - if(kde_resp_precalc.find(resp_size) == kde_resp_precalc.end()) - { - kde_resp = cv::Mat_((int)((resp_size / step_size)*(resp_size/step_size)), resp_size * resp_size); + if (kde_resp_precalc.find(resp_size) == kde_resp_precalc.end()) + { + kde_resp = cv::Mat_((int)((resp_size / step_size)*(resp_size / step_size)), resp_size * resp_size); cv::MatIterator_ kde_it = kde_resp.begin(); - for(int x = 0; x < resp_size/step_size; x++) + for (int x = 0; x < resp_size / step_size; x++) { float dx = x * step_size; - for(int y = 0; y < resp_size/step_size; y++) + for (int y = 0; y < resp_size / step_size; y++) { float dy = y * step_size; - int ii,jj; - float v,vx,vy; - - for(ii = 0; ii < resp_size; ii++) + int ii, jj; + float v, vx, vy; + + for (ii = 0; ii < resp_size; ii++) { - vx = (dy-ii)*(dy-ii); - for(jj = 0; jj < resp_size; jj++) + vx = (dy - ii)*(dy - ii); + for (jj = 0; jj < resp_size; jj++) { - vy = (dx-jj)*(dx-jj); + vy = (dx - jj)*(dx - jj); // the KDE evaluation of that point - v = exp(a*(vx+vy)); - + v = exp(a*(vx + vy)); + *kde_it++ = v; } } @@ -813,12 +799,12 @@ void CLNF::NonVectorisedMeanShift_precalc_kde(cv::Mat_& out_mean_shifts, } // for every point (patch) calculating mean-shift - for(int i = 0; i < n; i++) + for (int i = 0; i < n; i++) { - if(patch_experts.visibilities[scale][view_id].at(i,0) == 0) + if (patch_experts.visibilities[scale][view_id].at(i, 0) == 0) { - out_mean_shifts.at(i,0) = 0; - out_mean_shifts.at(i+n,0) = 0; + out_mean_shifts.at(i, 0) = 0; + out_mean_shifts.at(i + n, 0) = 0; continue; } @@ -827,33 +813,33 @@ void CLNF::NonVectorisedMeanShift_precalc_kde(cv::Mat_& out_mean_shifts, float dy = dys.at(i); // Ensure that we are within bounds (important for precalculation) - if(dx < 0) + if (dx < 0) dx = 0; - if(dy < 0) + if (dy < 0) dy = 0; - if(dx > resp_size - step_size) + if (dx > resp_size - step_size) dx = resp_size - step_size; - if(dy > resp_size - step_size) + if (dy > resp_size - step_size) dy = resp_size - step_size; - + // Pick the row from precalculated kde that approximates the current dx, dy best - int closest_col = (int)(dy /step_size + 0.5); // Plus 0.5 is there, as C++ rounds down with int cast - int closest_row = (int)(dx /step_size + 0.5); // Plus 0.5 is there, as C++ rounds down with int cast - - int idx = closest_row * ((int)(resp_size/step_size + 0.5)) + closest_col; // Plus 0.5 is there, as C++ rounds down with int cast + int closest_col = (int)(dy / step_size + 0.5); // Plus 0.5 is there, as C++ rounds down with int cast + int closest_row = (int)(dx / step_size + 0.5); // Plus 0.5 is there, as C++ rounds down with int cast + + int idx = closest_row * ((int)(resp_size / step_size + 0.5)) + closest_col; // Plus 0.5 is there, as C++ rounds down with int cast cv::MatIterator_ kde_it = kde_resp.begin() + kde_resp.cols*idx; - - float mx=0.0; - float my=0.0; - float sum=0.0; + + float mx = 0.0; + float my = 0.0; + float sum = 0.0; // Iterate over the patch responses here cv::MatConstIterator_ p = patch_expert_responses[i].begin(); - - for(int ii = 0; ii < resp_size; ii++) + + for (int ii = 0; ii < resp_size; ii++) { - for(int jj = 0; jj < resp_size; jj++) + for (int jj = 0; jj < resp_size; jj++) { // the KDE evaluation of that point multiplied by the probability at the current, xi, yi @@ -863,16 +849,16 @@ void CLNF::NonVectorisedMeanShift_precalc_kde(cv::Mat_& out_mean_shifts, // mean shift in x and y mx += v*jj; - my += v*ii; + my += v*ii; } } - - float msx = (mx/sum - dx); - float msy = (my/sum - dy); - out_mean_shifts.at(i,0) = msx; - out_mean_shifts.at(i+n,0) = msy; + float msx = (mx / sum - dx); + float msy = (my / sum - dy); + + out_mean_shifts.at(i, 0) = msx; + out_mean_shifts.at(i + n, 0) = msy; } @@ -880,61 +866,61 @@ void CLNF::NonVectorisedMeanShift_precalc_kde(cv::Mat_& out_mean_shifts, void CLNF::GetWeightMatrix(cv::Mat_& WeightMatrix, int scale, int view_id, const FaceModelParameters& parameters) { - int n = pdm.NumberOfPoints(); + int n = pdm.NumberOfPoints(); // Is the weight matrix needed at all - if(parameters.weight_factor > 0) + if (parameters.weight_factor > 0) { - WeightMatrix = cv::Mat_::zeros(n*2, n*2); + WeightMatrix = cv::Mat_::zeros(n * 2, n * 2); - for (int p=0; p < n; p++) + for (int p = 0; p < n; p++) { - if(!patch_experts.ccnf_expert_intensity.empty()) + if (!patch_experts.ccnf_expert_intensity.empty()) { // for the x dimension - WeightMatrix.at(p,p) = WeightMatrix.at(p,p) + patch_experts.ccnf_expert_intensity[scale][view_id][p].patch_confidence; - + WeightMatrix.at(p, p) = WeightMatrix.at(p, p) + patch_experts.ccnf_expert_intensity[scale][view_id][p].patch_confidence; + // for they y dimension - WeightMatrix.at(p+n,p+n) = WeightMatrix.at(p,p); + WeightMatrix.at(p + n, p + n) = WeightMatrix.at(p, p); } else { // Across the modalities add the confidences - for(size_t pc=0; pc < patch_experts.svr_expert_intensity[scale][view_id][p].svr_patch_experts.size(); pc++) + for (size_t pc = 0; pc < patch_experts.svr_expert_intensity[scale][view_id][p].svr_patch_experts.size(); pc++) { // for the x dimension - WeightMatrix.at(p,p) = WeightMatrix.at(p,p) + patch_experts.svr_expert_intensity[scale][view_id][p].svr_patch_experts.at(pc).confidence; - } + WeightMatrix.at(p, p) = WeightMatrix.at(p, p) + patch_experts.svr_expert_intensity[scale][view_id][p].svr_patch_experts.at(pc).confidence; + } // for the y dimension - WeightMatrix.at(p+n,p+n) = WeightMatrix.at(p,p); + WeightMatrix.at(p + n, p + n) = WeightMatrix.at(p, p); } } WeightMatrix = parameters.weight_factor * WeightMatrix; } else { - WeightMatrix = cv::Mat_::eye(n*2, n*2); + WeightMatrix = cv::Mat_::eye(n * 2, n * 2); } } //============================================================================= double CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const vector >& patch_expert_responses, const cv::Vec6d& initial_global, const cv::Mat_& initial_local, - const cv::Mat_& base_shape, const cv::Matx22d& sim_img_to_ref, const cv::Matx22f& sim_ref_to_img, int resp_size, int view_id, bool rigid, int scale, cv::Mat_& landmark_lhoods, - const FaceModelParameters& parameters) -{ + const cv::Mat_& base_shape, const cv::Matx22d& sim_img_to_ref, const cv::Matx22f& sim_ref_to_img, int resp_size, int view_id, bool rigid, int scale, cv::Mat_& landmark_lhoods, + const FaceModelParameters& parameters) +{ + + int n = pdm.NumberOfPoints(); - int n = pdm.NumberOfPoints(); - // Mean, eigenvalues, eigenvectors cv::Mat_ M = this->pdm.mean_shape; cv::Mat_ E = this->pdm.eigen_values; //Mat_ V = this->pdm.princ_comp; int m = pdm.NumberOfModes(); - + cv::Vec6d current_global(initial_global); cv::Mat_ current_local; @@ -946,9 +932,9 @@ double CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, con // Pre-calculate the regularisation term cv::Mat_ regTerm; - if(rigid) + if (rigid) { - regTerm = cv::Mat_::zeros(6,6); + regTerm = cv::Mat_::zeros(6, 6); } else { @@ -958,38 +944,38 @@ double CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, con cv::Mat(parameters.reg_factor / E).copyTo(regularisations(cv::Rect(6, 0, m, 1))); cv::Mat_ regTerm_d = cv::Mat::diag(regularisations.t()); regTerm_d.convertTo(regTerm, CV_32F); - } + } cv::Mat_ WeightMatrix; GetWeightMatrix(WeightMatrix, scale, view_id, parameters); cv::Mat_ dxs, dys; - + // The preallocated memory for the mean shifts cv::Mat_ mean_shifts(2 * pdm.NumberOfPoints(), 1, 0.0); // Number of iterations - for(int iter = 0; iter < parameters.num_optimisation_iteration; iter++) + for (int iter = 0; iter < parameters.num_optimisation_iteration; iter++) { // get the current estimates of x pdm.CalcShape2D(current_shape, current_local, current_global); - - if(iter > 0) + + if (iter > 0) { // if the shape hasn't changed terminate - if(norm(current_shape, previous_shape) < 0.01) - { + if (norm(current_shape, previous_shape) < 0.01) + { break; } } current_shape.copyTo(previous_shape); - + // Jacobian, and transposed weighted jacobian cv::Mat_ J, J_w_t; // calculate the appropriate Jacobians in 2D, even though the actual behaviour is in 3D, using small angle approximation and oriented shape - if(rigid) + if (rigid) { pdm.ComputeRigidJacobian(current_local, current_global, J, WeightMatrix, J_w_t); } @@ -997,39 +983,39 @@ double CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, con { pdm.ComputeJacobian(current_local, current_global, J, WeightMatrix, J_w_t); } - + // useful for mean shift calculation - float a = -0.5/(parameters.sigma * parameters.sigma); + float a = -0.5 / (parameters.sigma * parameters.sigma); cv::Mat_ current_shape_2D = current_shape.reshape(1, 2).t(); cv::Mat_ base_shape_2D = base_shape.reshape(1, 2).t(); cv::Mat_ offsets; cv::Mat((current_shape_2D - base_shape_2D) * cv::Mat(sim_img_to_ref).t()).convertTo(offsets, CV_32F); - - dxs = offsets.col(0) + (resp_size-1)/2; - dys = offsets.col(1) + (resp_size-1)/2; - + + dxs = offsets.col(0) + (resp_size - 1) / 2; + dys = offsets.col(1) + (resp_size - 1) / 2; + NonVectorisedMeanShift_precalc_kde(mean_shifts, patch_expert_responses, dxs, dys, resp_size, a, scale, view_id, kde_resp_precalc); // Now transform the mean shifts to the the image reference frame, as opposed to one of ref shape (object space) cv::Mat_ mean_shifts_2D = (mean_shifts.reshape(1, 2)).t(); - + mean_shifts_2D = mean_shifts_2D * cv::Mat(sim_ref_to_img).t(); - mean_shifts = cv::Mat(mean_shifts_2D.t()).reshape(1, n*2); + mean_shifts = cv::Mat(mean_shifts_2D.t()).reshape(1, n * 2); // remove non-visible observations - for(int i = 0; i < n; ++i) + for (int i = 0; i < n; ++i) { // if patch unavailable for current index - if(patch_experts.visibilities[scale][view_id].at(i,0) == 0) - { + if (patch_experts.visibilities[scale][view_id].at(i, 0) == 0) + { cv::Mat Jx = J.row(i); Jx = cvScalar(0); - cv::Mat Jy = J.row(i+n); + cv::Mat Jy = J.row(i + n); Jy = cvScalar(0); - mean_shifts.at(i,0) = 0.0f; - mean_shifts.at(i+n,0) = 0.0f; + mean_shifts.at(i, 0) = 0.0f; + mean_shifts.at(i + n, 0) = 0.0f; } } @@ -1037,9 +1023,9 @@ double CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, con cv::Mat_ J_w_t_m = J_w_t * mean_shifts; // Add the regularisation term - if(!rigid) + if (!rigid) { - J_w_t_m(cv::Rect(0,6,1, m)) = J_w_t_m(cv::Rect(0,6,1, m)) - regTerm(cv::Rect(6,6, m, m)) * current_local; + J_w_t_m(cv::Rect(0, 6, 1, m)) = J_w_t_m(cv::Rect(0, 6, 1, m)) - regTerm(cv::Rect(6, 6, m, m)) * current_local; } // Calculating the Hessian approximation @@ -1051,10 +1037,10 @@ double CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, con // Solve for the parameter update (from Baltrusaitis 2013 based on eq (36) Saragih 2011) cv::Mat_ param_update; cv::solve(Hessian, J_w_t_m, param_update, CV_CHOLESKY); - + // update the reference - pdm.UpdateModelParameters(param_update, current_local, current_global); - + pdm.UpdateModelParameters(param_update, current_local, current_global); + // clamp to the local parameters for valid expressions pdm.Clamp(current_local, current_global, parameters); @@ -1062,48 +1048,48 @@ double CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, con // compute the log likelihood double loglhood = 0; - + landmark_lhoods = cv::Mat_(n, 1, -1e8); - - for(int i = 0; i < n; i++) + + for (int i = 0; i < n; i++) { - if(patch_experts.visibilities[scale][view_id].at(i,0) == 0 ) + if (patch_experts.visibilities[scale][view_id].at(i, 0) == 0) { continue; } float dx = dxs.at(i); float dy = dys.at(i); - int ii,jj; - float v,vx,vy,sum=0.0; + int ii, jj; + float v, vx, vy, sum = 0.0; // Iterate over the patch responses here cv::MatConstIterator_ p = patch_expert_responses[i].begin(); - - for(ii = 0; ii < resp_size; ii++) + + for (ii = 0; ii < resp_size; ii++) { - vx = (dy-ii)*(dy-ii); - for(jj = 0; jj < resp_size; jj++) + vx = (dy - ii)*(dy - ii); + for (jj = 0; jj < resp_size; jj++) { - vy = (dx-jj)*(dx-jj); + vy = (dx - jj)*(dx - jj); // the probability at the current, xi, yi v = *p++; // the KDE evaluation of that point - v *= exp(-0.5*(vx+vy)/(parameters.sigma * parameters.sigma)); + v *= exp(-0.5*(vx + vy) / (parameters.sigma * parameters.sigma)); sum += v; } } - landmark_lhoods.at(i,0) = (double)sum; + landmark_lhoods.at(i, 0) = (double)sum; // the offset is there for numerical stability loglhood += log(sum + 1e-8); - } - loglhood = loglhood/sum(patch_experts.visibilities[scale][view_id])[0]; + } + loglhood = loglhood / sum(patch_experts.visibilities[scale][view_id])[0]; final_global = current_global; final_local = current_local; @@ -1115,50 +1101,56 @@ double CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, con // Getting a 3D shape model from the current detected landmarks (in camera space) cv::Mat_ CLNF::GetShape(double fx, double fy, double cx, double cy) const { - int n = this->detected_landmarks.rows/2; + int n = this->detected_landmarks.rows / 2; - cv::Mat_ shape3d(n*3, 1); + cv::Mat_ outShape(n, 3, 0.0); - this->pdm.CalcShape3D(shape3d, this->params_local); - - // Need to rotate the shape to get the actual 3D representation - - // get the rotation matrix from the euler angles - cv::Matx33d R = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(params_global[1], params_global[2], params_global[3])); - - shape3d = shape3d.reshape(1, 3); - - shape3d = shape3d.t() * cv::Mat(R).t(); - - // from the weak perspective model can determine the average depth of the object - double Zavg = fx / params_global[0]; - - cv::Mat_ outShape(n,3,0.0); - - // this is described in the paper in section 3.4 (equation 10) (of the CLM-Z paper) - for(int i = 0; i < n; i++) + // If the tracking started (otherwise no point reporting 3D shape) + if (this->IsInitialized()) { - double Z = Zavg + shape3d.at(i,2); - double X = Z * ((this->detected_landmarks.at(i) - cx)/fx); - double Y = Z * ((this->detected_landmarks.at(i + n) - cy)/fy); + cv::Mat_ shape3d(n * 3, 1); - outShape.at(i,0) = (double)X; - outShape.at(i,1) = (double)Y; - outShape.at(i,2) = (double)Z; + this->pdm.CalcShape3D(shape3d, this->params_local); + // Need to rotate the shape to get the actual 3D representation + + // get the rotation matrix from the euler angles + cv::Matx33d R = Utilities::Euler2RotationMatrix(cv::Vec3d(params_global[1], params_global[2], params_global[3])); + + shape3d = shape3d.reshape(1, 3); + + shape3d = shape3d.t() * cv::Mat(R).t(); + + // from the weak perspective model can determine the average depth of the object + double Zavg = fx / params_global[0]; + + + // this is described in the paper in section 3.4 (equation 10) (of the CLM-Z paper) + for (int i = 0; i < n; i++) + { + double Z = Zavg + shape3d.at(i, 2); + + double X = Z * ((this->detected_landmarks.at(i) - cx) / fx); + double Y = Z * ((this->detected_landmarks.at(i + n) - cy) / fy); + + outShape.at(i, 0) = (double)X; + outShape.at(i, 1) = (double)Y; + outShape.at(i, 2) = (double)Z; + + } } // The format is 3 rows - n cols return outShape.t(); - + } // A utility bounding box function cv::Rect_ CLNF::GetBoundingBox() const { - cv::Mat_ xs = this->detected_landmarks(cv::Rect(0,0,1,this->detected_landmarks.rows/2)); - cv::Mat_ ys = this->detected_landmarks(cv::Rect(0,this->detected_landmarks.rows/2, 1, this->detected_landmarks.rows/2)); + cv::Mat_ xs = this->detected_landmarks(cv::Rect(0, 0, 1, this->detected_landmarks.rows / 2)); + cv::Mat_ ys = this->detected_landmarks(cv::Rect(0, this->detected_landmarks.rows / 2, 1, this->detected_landmarks.rows / 2)); double min_x, max_x; double min_y, max_y; @@ -1173,16 +1165,16 @@ cv::Rect_ CLNF::GetBoundingBox() const // Legacy function not used at the moment void CLNF::NonVectorisedMeanShift(cv::Mat_& out_mean_shifts, const vector >& patch_expert_responses, const cv::Mat_ &dxs, const cv::Mat_ &dys, int resp_size, double a, int scale, int view_id) { - + int n = dxs.rows; - - for(int i = 0; i < n; i++) + + for (int i = 0; i < n; i++) { - if(patch_experts.visibilities[scale][view_id].at(i,0) == 0 || sum(patch_expert_responses[i])[0] == 0) + if (patch_experts.visibilities[scale][view_id].at(i, 0) == 0 || sum(patch_expert_responses[i])[0] == 0) { - out_mean_shifts.at(i,0) = 0; - out_mean_shifts.at(i+n,0) = 0; + out_mean_shifts.at(i, 0) = 0; + out_mean_shifts.at(i + n, 0) = 0; continue; } @@ -1190,41 +1182,41 @@ void CLNF::NonVectorisedMeanShift(cv::Mat_& out_mean_shifts, const vecto double dx = dxs.at(i); double dy = dys.at(i); - int ii,jj; - double v,vx,vy,mx=0.0,my=0.0,sum=0.0; + int ii, jj; + double v, vx, vy, mx = 0.0, my = 0.0, sum = 0.0; // Iterate over the patch responses here cv::MatConstIterator_ p = patch_expert_responses[i].begin(); - - for(ii = 0; ii < resp_size; ii++) + + for (ii = 0; ii < resp_size; ii++) { - vx = (dy-ii)*(dy-ii); - for(jj = 0; jj < resp_size; jj++) + vx = (dy - ii)*(dy - ii); + for (jj = 0; jj < resp_size; jj++) { - vy = (dx-jj)*(dx-jj); + vy = (dx - jj)*(dx - jj); // the probability at the current, xi, yi v = *p++; // the KDE evaluation of that point - double kd = exp(a*(vx+vy)); + double kd = exp(a*(vx + vy)); v *= kd; sum += v; // mean shift in x and y mx += v*jj; - my += v*ii; + my += v*ii; } } // setting the actual mean shift update - double msx = (mx/sum - dx); - double msy = (my/sum - dy); + double msx = (mx / sum - dx); + double msy = (my / sum - dy); out_mean_shifts.at(i, 0) = msx; out_mean_shifts.at(i + n, 0) = msy; - + } } diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp index 96aa02a..798a2c4 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp @@ -148,12 +148,6 @@ FaceModelParameters::FaceModelParameters(vector &arguments) valid[i + 1] = false; i++; } - else if (arguments[i].compare("-gaze") == 0) - { - track_gaze = true; - - valid[i] = false; - } else if (arguments[i].compare("-q") == 0) { @@ -253,7 +247,7 @@ void FaceModelParameters::init() reg_factor = 25; weight_factor = 0; // By default do not use NU-RLMS for videos as it does not work as well for them - validation_boundary = -0.45; + validation_boundary = 0.725; limit_pose = true; multi_view = false; @@ -267,7 +261,5 @@ void FaceModelParameters::init() // By default use HOG SVM curr_face_detector = HOG_SVM_DETECTOR; - // The gaze tracking has to be explicitly initialised - track_gaze = false; } diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 51bb369..0f2ba56 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -39,415 +39,12 @@ // OpenCV includes #include #include -#include - -// Boost includes -#include -#include - -using namespace boost::filesystem; using namespace std; namespace LandmarkDetector { -// For subpixel accuracy drawing -const int draw_shiftbits = 4; -const int draw_multiplier = 1 << 4; - - -// Useful utility for creating directories for storing the output files -void create_directory_from_file(string output_path) -{ - - // Creating the right directory structure - - // First get rid of the file - auto p = path(path(output_path).parent_path()); - - if(!p.empty() && !boost::filesystem::exists(p)) - { - bool success = boost::filesystem::create_directories(p); - if(!success) - { - cout << "Failed to create a directory... " << p.string() << endl; - } - } -} - -// Useful utility for creating directories for storing the output files -void create_directories(string output_path) -{ - - // Creating the right directory structure - - // First get rid of the file - auto p = path(output_path); - - if(!p.empty() && !boost::filesystem::exists(p)) - { - bool success = boost::filesystem::create_directories(p); - if(!success) - { - cout << "Failed to create a directory... " << p.string() << endl; - } - } -} - -// Extracting the following command line arguments -f, -op, -of, -ov (and possible ordered repetitions) -void get_video_input_output_params(vector &input_video_files, vector &output_files, - vector &output_video_files, string& output_codec, vector &arguments) -{ - bool* valid = new bool[arguments.size()]; - - for(size_t i = 0; i < arguments.size(); ++i) - { - valid[i] = true; - } - - // By default use DIVX codec - output_codec = "DIVX"; - - string input_root = ""; - string output_root = ""; - - string separator = string(1, boost::filesystem::path::preferred_separator); - - // First check if there is a root argument (so that videos and outputs could be defined more easilly) - for(size_t i = 0; i < arguments.size(); ++i) - { - if (arguments[i].compare("-root") == 0) - { - input_root = arguments[i + 1] + separator; - output_root = arguments[i + 1] + separator; - - // Add the / or \ to the directory - i++; - } - if (arguments[i].compare("-inroot") == 0) - { - input_root = arguments[i + 1] + separator; - i++; - } - if (arguments[i].compare("-outroot") == 0) - { - output_root = arguments[i + 1] + separator; - i++; - } - } - - for(size_t i = 0; i < arguments.size(); ++i) - { - if (arguments[i].compare("-f") == 0) - { - input_video_files.push_back(input_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-of") == 0) - { - output_files.push_back(output_root + arguments[i + 1]); - create_directory_from_file(output_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-ov") == 0) - { - output_video_files.push_back(output_root + arguments[i + 1]); - create_directory_from_file(output_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-oc") == 0) - { - if(arguments[i + 1].length() == 4) - output_codec = arguments[i + 1]; - i++; - } - } - - for(int i=arguments.size()-1; i >= 0; --i) - { - if(!valid[i]) - { - arguments.erase(arguments.begin()+i); - } - } - -} - -void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector &arguments) -{ - bool* valid = new bool[arguments.size()]; - - for(size_t i=0; i < arguments.size(); ++i) - { - valid[i] = true; - if (arguments[i].compare("-fx") == 0) - { - stringstream data(arguments[i+1]); - data >> fx; - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-fy") == 0) - { - stringstream data(arguments[i+1]); - data >> fy; - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-cx") == 0) - { - stringstream data(arguments[i+1]); - data >> cx; - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-cy") == 0) - { - stringstream data(arguments[i+1]); - data >> cy; - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-device") == 0) - { - stringstream data(arguments[i+1]); - data >> device; - valid[i] = false; - valid[i+1] = false; - i++; - } - } - - for(int i=arguments.size()-1; i >= 0; --i) - { - if(!valid[i]) - { - arguments.erase(arguments.begin()+i); - } - } -} - -void get_image_input_output_params(vector &input_image_files, vector &output_feature_files, vector &output_pose_files, vector &output_image_files, - vector> &input_bounding_boxes, vector &arguments) -{ - bool* valid = new bool[arguments.size()]; - - string out_pts_dir, out_pose_dir, out_img_dir; - - string input_root = ""; - string output_root = ""; - - string separator = string(1, boost::filesystem::path::preferred_separator); - - // First check if there is a root argument (so that videos and outputs could be defined more easilly) - for (size_t i = 0; i < arguments.size(); ++i) - { - if (arguments[i].compare("-root") == 0) - { - input_root = arguments[i + 1] + separator; - output_root = arguments[i + 1] + separator; - i++; - } - if (arguments[i].compare("-inroot") == 0) - { - input_root = arguments[i + 1] + separator; - i++; - } - if (arguments[i].compare("-outroot") == 0) - { - output_root = arguments[i + 1] + separator; - i++; - } - } - - for(size_t i = 0; i < arguments.size(); ++i) - { - valid[i] = true; - if (arguments[i].compare("-f") == 0) - { - input_image_files.push_back(input_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-fdir") == 0) - { - - // parse the -fdir directory by reading in all of the .png and .jpg files in it - path image_directory (arguments[i+1]); - - try - { - // does the file exist and is it a directory - if (exists(image_directory) && is_directory(image_directory)) - { - - vector file_in_directory; - copy(directory_iterator(image_directory), directory_iterator(), back_inserter(file_in_directory)); - - // Sort the images in the directory first - sort(file_in_directory.begin(), file_in_directory.end()); - - for (vector::const_iterator file_iterator (file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator) - { - // Possible image extension .jpg and .png - if(file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".png") == 0 || file_iterator->extension().string().compare(".bmp") == 0) - { - - - input_image_files.push_back(file_iterator->string()); - - // If there exists a .txt file corresponding to the image, it is assumed that it contains a bounding box definition for a face - // [minx, miny, maxx, maxy] - path current_file = *file_iterator; - path bbox = current_file.replace_extension("txt"); - - // If there is a bounding box file push it to the list of bounding boxes - if(exists(bbox)) - { - - std::ifstream in_bbox(bbox.string().c_str(), ios_base::in); - - double min_x, min_y, max_x, max_y; - - in_bbox >> min_x >> min_y >> max_x >> max_y; - - in_bbox.close(); - - input_bounding_boxes.push_back(cv::Rect_(min_x, min_y, max_x - min_x, max_y - min_y)); - } - } - } - } - } - catch (const filesystem_error& ex) - { - cout << ex.what() << '\n'; - } - - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-ofdir") == 0) - { - out_pts_dir = arguments[i + 1]; - create_directories(out_pts_dir); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-opdir") == 0) - { - out_pose_dir = arguments[i + 1]; - create_directories(out_pose_dir); - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-oidir") == 0) - { - out_img_dir = arguments[i + 1]; - create_directories(out_img_dir); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-op") == 0) - { - output_pose_files.push_back(output_root + arguments[i + 1]); - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-of") == 0) - { - output_feature_files.push_back(output_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-oi") == 0) - { - output_image_files.push_back(output_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - } - - // If any output directories are defined populate them based on image names - if(!out_img_dir.empty()) - { - for(size_t i=0; i < input_image_files.size(); ++i) - { - path image_loc(input_image_files[i]); - - path fname = image_loc.filename(); - fname = fname.replace_extension("bmp"); - output_image_files.push_back(out_img_dir + "/" + fname.string()); - - } - if(!input_image_files.empty()) - { - create_directory_from_file(output_image_files[0]); - } - } - - if(!out_pts_dir.empty()) - { - for(size_t i=0; i < input_image_files.size(); ++i) - { - path image_loc(input_image_files[i]); - - path fname = image_loc.filename(); - fname = fname.replace_extension("pts"); - output_feature_files.push_back(out_pts_dir + "/" + fname.string()); - } - create_directory_from_file(output_feature_files[0]); - } - - if (!out_pose_dir.empty()) - { - for (size_t i = 0; i < input_image_files.size(); ++i) - { - path image_loc(input_image_files[i]); - - path fname = image_loc.filename(); - fname = fname.replace_extension("pose"); - output_pose_files.push_back(out_pose_dir + "/" + fname.string()); - } - create_directory_from_file(output_pose_files[0]); - } - - // Make sure the same number of images and bounding boxes is present, if any bounding boxes are defined - if(input_bounding_boxes.size() > 0) - { - assert(input_bounding_boxes.size() == input_image_files.size()); - } - - // Clear up the argument list - for(int i=arguments.size()-1; i >= 0; --i) - { - if(!valid[i]) - { - arguments.erase(arguments.begin()+i); - } - } - -} - //=========================================================================== // Fast patch expert response computation (linear model across a ROI) using normalised cross-correlation //=========================================================================== @@ -772,196 +369,6 @@ cv::Matx22d AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst) //=========================================================================== // Visualisation functions //=========================================================================== -void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy) -{ - dest = cv::Mat_(mesh.rows,2, 0.0); - - int num_points = mesh.rows; - - double X, Y, Z; - - - cv::Mat_::const_iterator mData = mesh.begin(); - cv::Mat_::iterator projected = dest.begin(); - - for(int i = 0;i < num_points; i++) - { - // Get the points - X = *(mData++); - Y = *(mData++); - Z = *(mData++); - - double x; - double y; - - // if depth is 0 the projection is different - if(Z != 0) - { - x = ((X * fx / Z) + cx); - y = ((Y * fy / Z) + cy); - } - else - { - x = X; - y = Y; - } - - // Project and store in dest matrix - (*projected++) = x; - (*projected++) = y; - } - -} - -void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy) -{ - double boxVerts[] = {-1, 1, -1, - 1, 1, -1, - 1, 1, 1, - -1, 1, 1, - 1, -1, 1, - 1, -1, -1, - -1, -1, -1, - -1, -1, 1}; - - vector> edges; - edges.push_back(pair(0,1)); - edges.push_back(pair(1,2)); - edges.push_back(pair(2,3)); - edges.push_back(pair(0,3)); - edges.push_back(pair(2,4)); - edges.push_back(pair(1,5)); - edges.push_back(pair(0,6)); - edges.push_back(pair(3,7)); - edges.push_back(pair(6,5)); - edges.push_back(pair(5,4)); - edges.push_back(pair(4,7)); - edges.push_back(pair(7,6)); - - // The size of the head is roughly 200mm x 200mm x 200mm - cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100; - - cv::Matx33d rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5])); - cv::Mat_ rotBox; - - // Rotate the box - rotBox = cv::Mat(rot) * box.t(); - rotBox = rotBox.t(); - - // Move the bounding box to head position - rotBox.col(0) = rotBox.col(0) + pose[0]; - rotBox.col(1) = rotBox.col(1) + pose[1]; - rotBox.col(2) = rotBox.col(2) + pose[2]; - - // draw the lines - cv::Mat_ rotBoxProj; - Project(rotBoxProj, rotBox, fx, fy, cx, cy); - - cv::Rect image_rect(0,0,image.cols * draw_multiplier, image.rows * draw_multiplier); - - for (size_t i = 0; i < edges.size(); ++i) - { - cv::Mat_ begin; - cv::Mat_ end; - - rotBoxProj.row(edges[i].first).copyTo(begin); - rotBoxProj.row(edges[i].second).copyTo(end); - - - cv::Point p1(cvRound(begin.at(0) * (double)draw_multiplier), cvRound(begin.at(1) * (double)draw_multiplier)); - cv::Point p2(cvRound(end.at(0) * (double)draw_multiplier), cvRound(end.at(1) * (double)draw_multiplier)); - - // Only draw the line if one of the points is inside the image - if(p1.inside(image_rect) || p2.inside(image_rect)) - { - cv::line(image, p1, p2, color, thickness, CV_AA, draw_shiftbits); - } - - } - -} - -vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy) -{ - double boxVerts[] = {-1, 1, -1, - 1, 1, -1, - 1, 1, 1, - -1, 1, 1, - 1, -1, 1, - 1, -1, -1, - -1, -1, -1, - -1, -1, 1}; - - vector> edges; - edges.push_back(pair(0,1)); - edges.push_back(pair(1,2)); - edges.push_back(pair(2,3)); - edges.push_back(pair(0,3)); - edges.push_back(pair(2,4)); - edges.push_back(pair(1,5)); - edges.push_back(pair(0,6)); - edges.push_back(pair(3,7)); - edges.push_back(pair(6,5)); - edges.push_back(pair(5,4)); - edges.push_back(pair(4,7)); - edges.push_back(pair(7,6)); - - // The size of the head is roughly 200mm x 200mm x 200mm - cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100; - - cv::Matx33d rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5])); - cv::Mat_ rotBox; - - // Rotate the box - rotBox = cv::Mat(rot) * box.t(); - rotBox = rotBox.t(); - - // Move the bounding box to head position - rotBox.col(0) = rotBox.col(0) + pose[0]; - rotBox.col(1) = rotBox.col(1) + pose[1]; - rotBox.col(2) = rotBox.col(2) + pose[2]; - - // draw the lines - cv::Mat_ rotBoxProj; - Project(rotBoxProj, rotBox, fx, fy, cx, cy); - - vector> lines; - - for (size_t i = 0; i < edges.size(); ++i) - { - cv::Mat_ begin; - cv::Mat_ end; - - rotBoxProj.row(edges[i].first).copyTo(begin); - rotBoxProj.row(edges[i].second).copyTo(end); - - cv::Point2d p1(begin.at(0), begin.at(1)); - cv::Point2d p2(end.at(0), end.at(1)); - - lines.push_back(pair(p1,p2)); - - } - - return lines; -} - -void DrawBox(vector> lines, cv::Mat image, cv::Scalar color, int thickness) -{ - cv::Rect image_rect(0,0,image.cols, image.rows); - - for (size_t i = 0; i < lines.size(); ++i) - { - cv::Point p1 = lines.at(i).first; - cv::Point p2 = lines.at(i).second; - // Only draw the line if one of the points is inside the image - if(p1.inside(image_rect) || p2.inside(image_rect)) - { - cv::line(image, p1, p2, color, thickness, CV_AA); - } - - } - -} // Computing landmarks (to be drawn later possibly) vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities) @@ -1029,7 +436,7 @@ vector CalculateVisibleLandmarks(const CLNF& clnf_model) if (clnf_model.detection_success) { int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0); - // Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation + // Because we may want to draw visible points, need to find which points patch experts consider visible at a certain orientation return CalculateVisibleLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]); } else @@ -1038,12 +445,12 @@ vector CalculateVisibleLandmarks(const CLNF& clnf_model) } } -// Computing eye landmarks (to be drawn later or in different interfaces) +// Computing eye landmarks vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model) { vector to_return; - // If the model has hierarchical updates draw those too + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) { @@ -1061,13 +468,38 @@ vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model) return to_return; } +// Computing the 3D eye landmarks +vector Calculate3DEyeLandmarks(const CLNF& clnf_model, double fx, double fy, double cx, double cy) +{ -// Computing eye landmarks (to be drawn later or in different interfaces) + vector to_return; + + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + { + + if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 || + clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0) + { + + auto lmks = clnf_model.hierarchical_models[i].GetShape(fx, fy, cx, cy); + + int num_landmarks = lmks.cols; + + for (int lmk = 0; lmk < num_landmarks; ++lmk) + { + cv::Point3d curr_lmk(lmks.at(0, lmk), lmks.at(1, lmk), lmks.at(2, lmk)); + to_return.push_back(curr_lmk); + } + } + } + return to_return; +} +// Computing eye landmarks vector CalculateAllEyeLandmarks(const CLNF& clnf_model) { vector to_return; - // If the model has hierarchical updates draw those too + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) { @@ -1085,232 +517,12 @@ vector CalculateAllEyeLandmarks(const CLNF& clnf_model) return to_return; } - -// Drawing landmarks on a face image -void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities) -{ - int n = shape2D.rows/2; - - - // Drawing feature points - if(n >= 66) - { - for( int i = 0; i < n; ++i) - { - if(visibilities.at(i)) - { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - - // A rough heuristic for drawn point size - int thickness = (int)std::ceil(3.0* ((double)img.cols) / 640.0); - int thickness_2 = (int)std::ceil(1.0* ((double)img.cols) / 640.0); - - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - - } - } - } - else if(n == 28) // drawing eyes - { - for( int i = 0; i < n; ++i) - { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - - // A rough heuristic for drawn point size - int thickness = 1.0; - int thickness_2 = 1.0; - - int next_point = i + 1; - if(i == 7) - next_point = 0; - if(i == 19) - next_point = 8; - if(i == 27) - next_point = 20; - - cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); - if( i < 8 || i > 19) - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - else - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA, draw_shiftbits); - - - } - } - else if(n == 6) - { - for( int i = 0; i < n; ++i) - { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - - // A rough heuristic for drawn point size - int thickness = 1.0; - int thickness_2 = 1.0; - - int next_point = i + 1; - if(i == 5) - next_point = 0; - - cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - } - } -} - -// Drawing landmarks on a face image -void Draw(cv::Mat img, const cv::Mat_& shape2D) -{ - - int n; - - if(shape2D.cols == 2) - { - n = shape2D.rows; - } - else if(shape2D.cols == 1) - { - n = shape2D.rows/2; - } - - for( int i = 0; i < n; ++i) - { - cv::Point featurePoint; - if(shape2D.cols == 1) - { - featurePoint = cv::Point(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - } - else - { - featurePoint = cv::Point(cvRound(shape2D.at(i, 0) * (double)draw_multiplier), cvRound(shape2D.at(i, 1) * (double)draw_multiplier)); - } - // A rough heuristic for drawn point size - int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); - int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0); - - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - - } - -} - -// Drawing detected landmarks on a face image -void Draw(cv::Mat img, const CLNF& clnf_model) -{ - - int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0); - - // Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation - Draw(img, clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]); - - // If the model has hierarchical updates draw those too - for(size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) - { - if(clnf_model.hierarchical_models[i].pdm.NumberOfPoints() != clnf_model.hierarchical_mapping[i].size()) - { - Draw(img, clnf_model.hierarchical_models[i]); - } - } -} - -void DrawLandmarks(cv::Mat img, vector landmarks) -{ - for(cv::Point p : landmarks) - { - - // A rough heuristic for drawn point size - int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); - int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0); - - cv::circle(img, p, 1, cv::Scalar(0,0,255), thickness, CV_AA); - cv::circle(img, p, 1, cv::Scalar(255,0,0), thickness_2, CV_AA); - } - -} - -//=========================================================================== -// Angle representation conversion helpers -//=========================================================================== - -// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign -cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles) -{ - cv::Matx33d rotation_matrix; - - double s1 = sin(eulerAngles[0]); - double s2 = sin(eulerAngles[1]); - double s3 = sin(eulerAngles[2]); - - double c1 = cos(eulerAngles[0]); - double c2 = cos(eulerAngles[1]); - double c3 = cos(eulerAngles[2]); - - rotation_matrix(0,0) = c2 * c3; - rotation_matrix(0,1) = -c2 *s3; - rotation_matrix(0,2) = s2; - rotation_matrix(1,0) = c1 * s3 + c3 * s1 * s2; - rotation_matrix(1,1) = c1 * c3 - s1 * s2 * s3; - rotation_matrix(1,2) = -c2 * s1; - rotation_matrix(2,0) = s1 * s3 - c1 * c3 * s2; - rotation_matrix(2,1) = c3 * s1 + c1 * s2 * s3; - rotation_matrix(2,2) = c1 * c2; - - return rotation_matrix; -} - -// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign -cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix) -{ - double q0 = sqrt( 1 + rotation_matrix(0,0) + rotation_matrix(1,1) + rotation_matrix(2,2) ) / 2.0; - double q1 = (rotation_matrix(2,1) - rotation_matrix(1,2)) / (4.0*q0) ; - double q2 = (rotation_matrix(0,2) - rotation_matrix(2,0)) / (4.0*q0) ; - double q3 = (rotation_matrix(1,0) - rotation_matrix(0,1)) / (4.0*q0) ; - - double t1 = 2.0 * (q0*q2 + q1*q3); - - double yaw = asin(2.0 * (q0*q2 + q1*q3)); - double pitch= atan2(2.0 * (q0*q1-q2*q3), q0*q0-q1*q1-q2*q2+q3*q3); - double roll = atan2(2.0 * (q0*q3-q1*q2), q0*q0+q1*q1-q2*q2-q3*q3); - - return cv::Vec3d(pitch, yaw, roll); -} - -cv::Vec3d Euler2AxisAngle(const cv::Vec3d& euler) -{ - cv::Matx33d rotMatrix = LandmarkDetector::Euler2RotationMatrix(euler); - cv::Vec3d axis_angle; - cv::Rodrigues(rotMatrix, axis_angle); - return axis_angle; -} - -cv::Vec3d AxisAngle2Euler(const cv::Vec3d& axis_angle) -{ - cv::Matx33d rotation_matrix; - cv::Rodrigues(axis_angle, rotation_matrix); - return RotationMatrix2Euler(rotation_matrix); -} - -cv::Matx33d AxisAngle2RotationMatrix(const cv::Vec3d& axis_angle) -{ - cv::Matx33d rotation_matrix; - cv::Rodrigues(axis_angle, rotation_matrix); - return rotation_matrix; -} - -cv::Vec3d RotationMatrix2AxisAngle(const cv::Matx33d& rotation_matrix) -{ - cv::Vec3d axis_angle; - cv::Rodrigues(rotation_matrix, axis_angle); - return axis_angle; -} - //=========================================================================== //============================================================================ // Face detection helpers //============================================================================ -bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, double min_width, cv::Rect_ roi) +bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity) { cv::CascadeClassifier classifier("./classifiers/haarcascade_frontalface_alt.xml"); if(classifier.empty()) @@ -1320,60 +532,47 @@ bool DetectFaces(vector >& o_regions, const cv::Mat_& i } else { - return DetectFaces(o_regions, intensity, classifier, min_width, roi); + return DetectFaces(o_regions, intensity, classifier); } } -bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, cv::CascadeClassifier& classifier, double min_width, cv::Rect_ roi) +bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, cv::CascadeClassifier& classifier) { vector face_detections; - if(min_width == -1) - { - classifier.detectMultiScale(intensity, face_detections, 1.2, 2, 0, cv::Size(50, 50)); - } - else - { - classifier.detectMultiScale(intensity, face_detections, 1.2, 2, 0, cv::Size(min_width, min_width)); - } + classifier.detectMultiScale(intensity, face_detections, 1.2, 2, 0, cv::Size(50, 50)); // Convert from int bounding box do a double one with corrections + o_regions.resize(face_detections.size()); + for( size_t face = 0; face < o_regions.size(); ++face) { // OpenCV is overgenerous with face size and y location is off // CLNF detector expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate // The scalings were learned using the Face Detections on LFPW, Helen, AFW and iBUG datasets, using ground truth and detections from openCV - cv::Rect_ region; + // Correct for scale - region.width = face_detections[face].width * 0.8924; - region.height = face_detections[face].height * 0.8676; + o_regions[face].width = face_detections[face].width * 0.8924; + o_regions[face].height = face_detections[face].height * 0.8676; // Move the face slightly to the right (as the width was made smaller) - region.x = face_detections[face].x + 0.0578 * face_detections[face].width; + o_regions[face].x = face_detections[face].x + 0.0578 * face_detections[face].width; // Shift face down as OpenCV Haar Cascade detects the forehead as well, and we're not interested - region.y = face_detections[face].y + face_detections[face].height * 0.2166; + o_regions[face].y = face_detections[face].y + face_detections[face].height * 0.2166; + - if (min_width != -1) - { - if (region.width < min_width || region.x < ((double)intensity.cols) * roi.x || region.y < ((double)intensity.cols) * roi.y || - region.x + region.width >((double)intensity.cols) * (roi.x + roi.width) || region.y + region.height >((double)intensity.rows) * (roi.y + roi.height)) - continue; - } - - - o_regions.push_back(region); } return o_regions.size() > 0; } -bool DetectSingleFace(cv::Rect_& o_region, const cv::Mat_& intensity_image, cv::CascadeClassifier& classifier, cv::Point preference, double min_width, cv::Rect_ roi) +bool DetectSingleFace(cv::Rect_& o_region, const cv::Mat_& intensity_image, cv::CascadeClassifier& classifier, cv::Point preference) { // The tracker can return multiple faces vector > face_detections; - bool detect_success = LandmarkDetector::DetectFaces(face_detections, intensity_image, classifier, min_width, roi); + bool detect_success = LandmarkDetector::DetectFaces(face_detections, intensity_image, classifier); if(detect_success) { @@ -1427,15 +626,15 @@ bool DetectSingleFace(cv::Rect_& o_region, const cv::Mat_& intens return detect_success; } -bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, std::vector& confidences, double min_width, cv::Rect_ roi) +bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, std::vector& confidences) { dlib::frontal_face_detector detector = dlib::get_frontal_face_detector(); - return DetectFacesHOG(o_regions, intensity, detector, confidences, min_width, roi); + return DetectFacesHOG(o_regions, intensity, detector, confidences); } -bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, dlib::frontal_face_detector& detector, std::vector& o_confidences, double min_width, cv::Rect_ roi) +bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, dlib::frontal_face_detector& detector, std::vector& o_confidences) { cv::Mat_ upsampled_intensity; @@ -1450,50 +649,39 @@ bool DetectFacesHOG(vector >& o_regions, const cv::Mat_ detector(cv_grayscale, face_detections, -0.2); // Convert from int bounding box do a double one with corrections - //o_regions.resize(face_detections.size()); - //o_confidences.resize(face_detections.size()); + o_regions.resize(face_detections.size()); + o_confidences.resize(face_detections.size()); - for( size_t face = 0; face < face_detections.size(); ++face) + for( size_t face = 0; face < o_regions.size(); ++face) { // CLNF expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate - cv::Rect_ region; + // The scalings were learned using the Face Detections on LFPW and Helen using ground truth and detections from the HOG detector + // Move the face slightly to the right (as the width was made smaller) - region.x = (face_detections[face].rect.get_rect().tl_corner().x() + 0.0389 * face_detections[face].rect.get_rect().width()) / scaling; + o_regions[face].x = (face_detections[face].rect.get_rect().tl_corner().x() + 0.0389 * face_detections[face].rect.get_rect().width())/scaling; // Shift face down as OpenCV Haar Cascade detects the forehead as well, and we're not interested - region.y = (face_detections[face].rect.get_rect().tl_corner().y() + 0.1278 * face_detections[face].rect.get_rect().height()) / scaling; + o_regions[face].y = (face_detections[face].rect.get_rect().tl_corner().y() + 0.1278 * face_detections[face].rect.get_rect().height())/scaling; // Correct for scale - region.width = (face_detections[face].rect.get_rect().width() * 0.9611) / scaling; - region.height = (face_detections[face].rect.get_rect().height() * 0.9388) / scaling; + o_regions[face].width = (face_detections[face].rect.get_rect().width() * 0.9611)/scaling; + o_regions[face].height = (face_detections[face].rect.get_rect().height() * 0.9388)/scaling; - // The scalings were learned using the Face Detections on LFPW and Helen using ground truth and detections from the HOG detector - if (min_width != -1) - { - if (region.width < min_width || region.x < ((double)intensity.cols) * roi.x || region.y < ((double)intensity.cols) * roi.y || - region.x + region.width > ((double)intensity.cols) * (roi.x+roi.width) || region.y + region.height >((double)intensity.rows) * (roi.y + roi.height)) - continue; - } - - - o_regions.push_back(region); - o_confidences.push_back(face_detections[face].detection_confidence); + o_confidences[face] = face_detections[face].detection_confidence; } return o_regions.size() > 0; } -bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& intensity_img, dlib::frontal_face_detector& detector, double& confidence, cv::Point preference, double min_width, cv::Rect_ roi) +bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& intensity_img, dlib::frontal_face_detector& detector, double& confidence, cv::Point preference) { // The tracker can return multiple faces vector > face_detections; vector confidences; - bool detect_success = LandmarkDetector::DetectFacesHOG(face_detections, intensity_img, detector, confidences, min_width, roi); - - // In case of multiple faces pick the biggest one - bool use_size = true; + bool detect_success = LandmarkDetector::DetectFacesHOG(face_detections, intensity_img, detector, confidences); + if(detect_success) { @@ -1502,14 +690,10 @@ bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& int // keep the most confident one or the one closest to preference point if set double best_so_far; if(use_preferred) - { + { best_so_far = sqrt((preference.x - (face_detections[0].width/2 + face_detections[0].x)) * (preference.x - (face_detections[0].width/2 + face_detections[0].x)) + (preference.y - (face_detections[0].height/2 + face_detections[0].y)) * (preference.y - (face_detections[0].height/2 + face_detections[0].y))); } - else if (use_size) - { - best_so_far = (face_detections[0].width + face_detections[0].height) / 2.0; - } else { best_so_far = confidences[0]; @@ -1528,11 +712,6 @@ bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& int (preference.y - (face_detections[i].height/2 + face_detections[i].y)) * (preference.y - (face_detections[i].height/2 + face_detections[i].y))); better = dist < best_so_far; } - else if (use_size) - { - dist = (face_detections[i].width + face_detections[i].height) / 2.0; - better = dist > best_so_far; - } else { dist = confidences[i]; diff --git a/lib/local/LandmarkDetector/src/PDM.cpp b/lib/local/LandmarkDetector/src/PDM.cpp index 74b0eef..4407535 100644 --- a/lib/local/LandmarkDetector/src/PDM.cpp +++ b/lib/local/LandmarkDetector/src/PDM.cpp @@ -49,6 +49,7 @@ #endif #include +#include "RotationHelpers.h" using namespace LandmarkDetector; //=========================================================================== @@ -154,7 +155,7 @@ void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& param // get the rotation matrix from the euler angles cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33d currRot = Euler2RotationMatrix(euler); + cv::Matx33d currRot = Utilities::Euler2RotationMatrix(euler); // get the 3D shape of the object cv::Mat_ Shape_3D = mean_shape + princ_comp * params_local; @@ -185,7 +186,7 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& boun CalcShape3D(current_shape, params_local); // rotate the shape - cv::Matx33d rotation_matrix = Euler2RotationMatrix(rotation); + cv::Matx33d rotation_matrix = Utilities::Euler2RotationMatrix(rotation); cv::Mat_ reshaped = current_shape.reshape(1, 3); @@ -265,7 +266,7 @@ void PDM::ComputeRigidJacobian(const cv::Mat_& p_local, const cv::Vec6d& // Get the rotation matrix cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33d currRot = Euler2RotationMatrix(euler); + cv::Matx33d currRot = Utilities::Euler2RotationMatrix(euler); float r11 = (float) currRot(0,0); float r12 = (float) currRot(0,1); @@ -363,7 +364,7 @@ void PDM::ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6d& shape_3D_d.convertTo(shape_3D, CV_32F); cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33d currRot = Euler2RotationMatrix(euler); + cv::Matx33d currRot = Utilities::Euler2RotationMatrix(euler); float r11 = (float) currRot(0,0); float r12 = (float) currRot(0,1); @@ -460,7 +461,7 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& // get the original rotation matrix cv::Vec3d eulerGlobal(params_global[1], params_global[2], params_global[3]); - cv::Matx33d R1 = Euler2RotationMatrix(eulerGlobal); + cv::Matx33d R1 = Utilities::Euler2RotationMatrix(eulerGlobal); // construct R' = [1, -wz, wy // wz, 1, -wx @@ -478,8 +479,8 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& cv::Matx33d R3 = R1 *R2; // Extract euler angle (through axis angle first to make sure it's legal) - cv::Vec3d axis_angle = RotationMatrix2AxisAngle(R3); - cv::Vec3d euler = AxisAngle2Euler(axis_angle); + cv::Vec3d axis_angle = Utilities::RotationMatrix2AxisAngle(R3); + cv::Vec3d euler = Utilities::AxisAngle2Euler(axis_angle); params_global[1] = euler[0]; params_global[2] = euler[1]; @@ -569,7 +570,7 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_ double scaling = ((width / model_bbox.width) + (height / model_bbox.height)) / 2; cv::Vec3d rotation_init = rotation; - cv::Matx33d R = Euler2RotationMatrix(rotation_init); + cv::Matx33d R = Utilities::Euler2RotationMatrix(rotation_init); cv::Vec2d translation((min_x + max_x) / 2.0, (min_y + max_y) / 2.0); cv::Mat_ loc_params(this->NumberOfModes(),1, 0.0); @@ -656,7 +657,7 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_ translation[0] = glob_params[4]; translation[1] = glob_params[5]; - R = Euler2RotationMatrix(rotation_init); + R = Utilities::Euler2RotationMatrix(rotation_init); R_2D(0,0) = R(0,0);R_2D(0,1) = R(0,1); R_2D(0,2) = R(0,2); R_2D(1,0) = R(1,0);R_2D(1,1) = R(1,1); R_2D(1,2) = R(1,2); diff --git a/lib/local/Utilities/CMakeLists.txt b/lib/local/Utilities/CMakeLists.txt new file mode 100644 index 0000000..73c39c0 --- /dev/null +++ b/lib/local/Utilities/CMakeLists.txt @@ -0,0 +1,31 @@ +include_directories(${BOOST_INCLUDE_DIR}) + +SET(SOURCE + src/ImageCapture.cpp + src/RecorderCSV.cpp + src/RecorderHOG.cpp + src/RecorderOpenFace.cpp + src/RecorderOpenFaceParameters.cpp + src/SequenceCapture.cpp + src/VisualizationUtils.cpp + src/Visualizer.cpp +) + +SET(HEADERS + include/ImageCapture.h + include/RecorderCSV.h + include/RecorderHOG.h + include/RecorderOpenFace.h + include/RecorderOpenFaceParameters.h + include/SequenceCapture.h + include/VisualizationUtils.h + include/Visualizer.h +) + +include_directories(./include) +include_directories(${UTILITIES_SOURCE_DIR}/include) + +add_library( Utilities ${SOURCE} ${HEADERS}) + +install (TARGETS Utilities DESTINATION lib) +install (FILES ${HEADERS} DESTINATION include/OpenFace) diff --git a/lib/local/Utilities/Utilities.vcxproj b/lib/local/Utilities/Utilities.vcxproj new file mode 100644 index 0000000..5ec041f --- /dev/null +++ b/lib/local/Utilities/Utilities.vcxproj @@ -0,0 +1,183 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + {8E741EA2-9386-4CF2-815E-6F9B08991EAC} + Win32Proj + Utilities + 8.1 + Utilities + + + + StaticLibrary + true + v140 + Unicode + + + StaticLibrary + false + v140 + true + Unicode + + + StaticLibrary + true + v140 + Unicode + + + StaticLibrary + false + v140 + true + Unicode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + ./include;%(AdditionalIncludeDirectories) + + + Windows + + + + + + + Level3 + Disabled + _DEBUG;_LIB;%(PreprocessorDefinitions) + ./include;%(AdditionalIncludeDirectories) + + + Windows + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + ./include;%(AdditionalIncludeDirectories) + + + Windows + true + true + + + + + Level3 + + + Full + + + false + NDEBUG;_LIB;%(PreprocessorDefinitions) + ./include;%(AdditionalIncludeDirectories) + Speed + AnySuitable + + + Windows + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + {b47a5f12-2567-44e9-ae49-35763ec82149} + + + + + + \ No newline at end of file diff --git a/lib/local/Utilities/Utilities.vcxproj.filters b/lib/local/Utilities/Utilities.vcxproj.filters new file mode 100644 index 0000000..eadc61c --- /dev/null +++ b/lib/local/Utilities/Utilities.vcxproj.filters @@ -0,0 +1,75 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + \ No newline at end of file diff --git a/lib/local/Utilities/include/ImageCapture.h b/lib/local/Utilities/include/ImageCapture.h new file mode 100644 index 0000000..f2a10e6 --- /dev/null +++ b/lib/local/Utilities/include/ImageCapture.h @@ -0,0 +1,121 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __IMAGE_CAPTURE_h_ +#define __IMAGE_CAPTURE_h_ + +// System includes +#include +#include +#include + +// OpenCV includes +#include +#include + +namespace Utilities +{ + + //=========================================================================== + /** + A class for capturing sequences from video, webcam, and image directories + */ + class ImageCapture { + + public: + + // Default constructor + ImageCapture() {}; + + // Opening based on command line arguments + bool Open(std::vector& arguments); + + // Direct opening + + // Image sequence in the directory + bool OpenDirectory(std::string directory, std::string bbox_directory="", float fx = -1, float fy = -1, float cx = -1, float cy = -1); + + // Video file + bool OpenImageFiles(const std::vector& image_files, float fx = -1, float fy = -1, float cx = -1, float cy = -1); + + // Getting the next frame + cv::Mat GetNextImage(); + + // Getting the most recent grayscale frame (need to call GetNextImage first) + cv::Mat_ GetGrayFrame(); + + // Return bounding boxes associated with the image (if defined) + std::vector > GetBoundingBoxes(); + + // Parameters describing the sequence and it's progress (what's the proportion of images opened) + double GetProgress(); + + int image_width; + int image_height; + + float fx, fy, cx, cy; + + // Name of the video file, image directory, or the webcam + std::string name; + + bool has_bounding_boxes; + + private: + + // Blocking copy and move, as it doesn't make sense to have several readers pointed at the same source + ImageCapture & operator= (const ImageCapture& other); + ImageCapture & operator= (const ImageCapture&& other); + ImageCapture(const ImageCapture&& other); + ImageCapture(const ImageCapture& other); + + // Storing the latest captures + cv::Mat latest_frame; + cv::Mat_ latest_gray_frame; + + // Keeping track of how many files are read and the filenames + size_t frame_num; + std::vector image_files; + + // Could optionally read the bounding box locations from files (each image could have multiple bounding boxes) + std::vector > > bounding_boxes; + + void SetCameraIntrinsics(float fx, float fy, float cx, float cy); + + bool image_focal_length_set; + bool image_optical_center_set; + + bool no_input_specified; + + }; +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/include/ImageManipulationHelpers.h b/lib/local/Utilities/include/ImageManipulationHelpers.h new file mode 100644 index 0000000..23a4cb0 --- /dev/null +++ b/lib/local/Utilities/include/ImageManipulationHelpers.h @@ -0,0 +1,91 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __IMAGE_MANIPULATION_HELPERS_h_ +#define __IMAGE_MANIPULATION_HELPERS_h_ + +#include +#include + +namespace Utilities +{ + //=========================================================================== + // Converting between color spaces and bit depths + //=========================================================================== + + static void ConvertToGrayscale_8bit(const cv::Mat& in, cv::Mat& out) + { + if (in.channels() == 3) + { + // Make sure it's in a correct format + if (in.depth() == CV_16U) + { + cv::Mat tmp = in / 256; + tmp.convertTo(out, CV_8U); + cv::cvtColor(out, out, CV_BGR2GRAY); + } + else + { + cv::cvtColor(in, out, CV_BGR2GRAY); + } + } + else if (in.channels() == 4) + { + if (in.depth() == CV_16U) + { + cv::Mat tmp = in / 256; + tmp.convertTo(out, CV_8U); + cv::cvtColor(out, out, CV_BGRA2GRAY); + } + else + { + cv::cvtColor(in, out, CV_BGRA2GRAY); + } + } + else + { + if (in.depth() == CV_16U) + { + cv::Mat tmp = in / 256; + tmp.convertTo(out, CV_8U); + } + else if (in.depth() == CV_8U) + { + out = in.clone(); + } + } + } + + +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/include/RecorderCSV.h b/lib/local/Utilities/include/RecorderCSV.h new file mode 100644 index 0000000..7a625d5 --- /dev/null +++ b/lib/local/Utilities/include/RecorderCSV.h @@ -0,0 +1,98 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __RECORDER_CSV_h_ +#define __RECORDER_CSV_h_ + +// System includes +#include +#include +#include + +// OpenCV includes +#include + +namespace Utilities +{ + + //=========================================================================== + /** + A class for recording CSV file from OpenFace + */ + class RecorderCSV { + + public: + + // The constructor for the recorder, need to specify if we are recording a sequence or not + RecorderCSV(); + + // Opening the file and preparing the header for it + bool Open(std::string output_file_name, bool is_sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, + int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector& au_names_class, const std::vector& au_names_reg); + + // Closing the file and cleaning up + void Close(); + + void WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence, + const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, const cv::Mat_& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate, + const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, + const std::vector >& au_intensities, const std::vector >& au_occurences); + + private: + + // Blocking copy and move, as it doesn't make sense to read to write to the same file + RecorderCSV & operator= (const RecorderCSV& other); + RecorderCSV & operator= (const RecorderCSV&& other); + RecorderCSV(const RecorderCSV&& other); + RecorderCSV(const RecorderCSV& other); + + // The actual output file stream that will be written + std::ofstream output_file; + + // If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face + bool is_sequence; + + // Keep track of what we are recording + bool output_2D_landmarks; + bool output_3D_landmarks; + bool output_model_params; + bool output_pose; + bool output_AUs; + bool output_gaze; + + std::vector au_names_class; + std::vector au_names_reg; + + }; +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/include/RecorderHOG.h b/lib/local/Utilities/include/RecorderHOG.h new file mode 100644 index 0000000..e091bff --- /dev/null +++ b/lib/local/Utilities/include/RecorderHOG.h @@ -0,0 +1,88 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __RECORDER_HOG_h_ +#define __RECORDER_HOG_h_ + +// System includes +#include + +// OpenCV includes +#include + +#include +#include + +namespace Utilities +{ + + //=========================================================================== + /** + A class for recording CSV file from OpenFace + */ + class RecorderHOG { + + public: + + // The constructor for the recorder, by default does not do anything + RecorderHOG(); + + // Adding observations to the recorder + void SetObservationHOG(bool success, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels); + + void Write(); + + bool Open(std::string filename); + + void Close(); + + private: + + // Blocking copy and move, as it doesn't make sense to read to write to the same file + RecorderHOG & operator= (const RecorderHOG& other); + RecorderHOG & operator= (const RecorderHOG&& other); + RecorderHOG(const RecorderHOG&& other); + RecorderHOG(const RecorderHOG& other); + + std::ofstream hog_file; + + // Internals for recording + int num_cols; + int num_rows; + int num_channels; + cv::Mat_ hog_descriptor; + bool good_frame; + + }; +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/include/RecorderOpenFace.h b/lib/local/Utilities/include/RecorderOpenFace.h new file mode 100644 index 0000000..25bdd13 --- /dev/null +++ b/lib/local/Utilities/include/RecorderOpenFace.h @@ -0,0 +1,160 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __RECORDER_OPENFACE_h_ +#define __RECORDER_OPENFACE_h_ + +#include "RecorderCSV.h" +#include "RecorderHOG.h" +#include "RecorderOpenFaceParameters.h" + +// System includes +#include + +// OpenCV includes +#include +#include + +namespace Utilities +{ + + //=========================================================================== + /** + A class for recording data processed by OpenFace (facial landmarks, head pose, facial action units, aligned face, HOG features, and tracked video + */ + class RecorderOpenFace { + + public: + + // The constructor for the recorder, need to specify if we are recording a sequence or not, in_filename should be just the name and not contain extensions + RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::vector& arguments); + + ~RecorderOpenFace(); + + // Closing and cleaning up the recorder + void Close(); + + // Adding observations to the recorder + + // Required observations for video/image-sequence + void SetObservationTimestamp(double timestamp); + + // All observations relevant to facial landmarks + void SetObservationLandmarks(const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, + const cv::Vec6d& params_global, const cv::Mat_& params_local, double confidence, bool success); + + // Pose related observations + void SetObservationPose(const cv::Vec6d& pose); + + // AU related observations + void SetObservationActionUnits(const std::vector >& au_intensities, + const std::vector >& au_occurences); + + // Gaze related observations + void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, + const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2D, const std::vector& eye_landmarks3D); + + // Face alignment related observations + void SetObservationFaceAlign(const cv::Mat& aligned_face); + + // HOG feature related observations + void SetObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels); + + void SetObservationVisualization(const cv::Mat &vis_track); + + void WriteObservation(); + + std::string GetCSVFile() { return csv_filename; } + + private: + + // Blocking copy, assignment and move operators, as it does not make sense to save to the same location + RecorderOpenFace & operator= (const RecorderOpenFace& other); + RecorderOpenFace & operator= (const RecorderOpenFace&& other); + RecorderOpenFace(const RecorderOpenFace&& other); + RecorderOpenFace(const RecorderOpenFace& other); + + // Keeping track of what to output and how to output it + const RecorderOpenFaceParameters params; + + // Keep track of the file and output root location + std::string record_root; + std::string default_record_directory = "processed"; // By default we are writing in the processed directory in the working directory, if no output parameters provided + std::string of_filename; + std::string filename; + std::string csv_filename; + std::string aligned_output_directory; + std::ofstream metadata_file; + + // The actual output file stream that will be written + RecorderCSV csv_recorder; + RecorderHOG hog_recorder; + + // The actual temporary storage for the observations + double timestamp; + + // Facial landmark related observations + cv::Mat_ landmarks_2D; + cv::Mat_ landmarks_3D; + cv::Vec6d pdm_params_global; + cv::Mat_ pdm_params_local; + double landmark_detection_confidence; + bool landmark_detection_success; + + // Head pose related observations + cv::Vec6d head_pose; + + // Action Unit related observations + std::vector > au_intensities; + std::vector > au_occurences; + + // Gaze related observations + cv::Point3f gaze_direction0; + cv::Point3f gaze_direction1; + cv::Vec2d gaze_angle; + std::vector eye_landmarks2D; + std::vector eye_landmarks3D; + + int observation_count; + + // For video writing + cv::VideoWriter video_writer; + std::string media_filename; + cv::Mat vis_to_out; + + // For aligned face writing + cv::Mat aligned_face; + + }; +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/include/RecorderOpenFaceParameters.h b/lib/local/Utilities/include/RecorderOpenFaceParameters.h new file mode 100644 index 0000000..d2cbf16 --- /dev/null +++ b/lib/local/Utilities/include/RecorderOpenFaceParameters.h @@ -0,0 +1,106 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +// Parameters of the Face analyser +#ifndef __RECORDER_OPENFACE_PARAM_H +#define __RECORDER_OPENFACE_PARAM_H + +#include +#include + +// Boost includes +#include +#include + +using namespace std; + +namespace Utilities +{ + + class RecorderOpenFaceParameters + { + + public: + + // Constructors + RecorderOpenFaceParameters(std::vector &arguments, bool sequence, bool is_from_webcam, float fx = -1, float fy = -1, float cx = -1, float cy = -1, double fps_vid_out = 30); + + bool isSequence() const { return is_sequence; } + bool isFromWebcam() const { return is_from_webcam; } + bool output2DLandmarks() const { return output_2D_landmarks; } + bool output3DLandmarks() const { return output_3D_landmarks; } + bool outputPDMParams() const { return output_model_params; } + bool outputPose() const { return output_pose; } + bool outputAUs() const { return output_AUs; } + bool outputGaze() const { return output_gaze; } + bool outputHOG() const { return output_hog; } + bool outputTracked() const { return output_tracked; } + bool outputAlignedFaces() const { return output_aligned_faces; } + std::string outputCodec() const { return output_codec; } + double outputFps() const { return fps_vid_out; } + + float getFx() const { return fx; } + float getFy() const { return fy; } + float getCx() const { return cx; } + float getCy() const { return cy; } + + private: + + // If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face + bool is_sequence; + // If the data is coming from a webcam + bool is_from_webcam; + + // Keep track of what we are recording + bool output_2D_landmarks; + bool output_3D_landmarks; + bool output_model_params; + bool output_pose; + bool output_AUs; + bool output_gaze; + bool output_hog; + bool output_tracked; + bool output_aligned_faces; + + // Some video recording parameters + std::string output_codec; + double fps_vid_out; + + // Camera parameters for recording in the meta file; + float fx, fy, cx, cy; + + }; + +} + +#endif // ____RECORDER_OPENFACE_PARAM_H diff --git a/lib/local/Utilities/include/RotationHelpers.h b/lib/local/Utilities/include/RotationHelpers.h new file mode 100644 index 0000000..242a0a1 --- /dev/null +++ b/lib/local/Utilities/include/RotationHelpers.h @@ -0,0 +1,160 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __ROTATION_HELPERS_h_ +#define __ROTATION_HELPERS_h_ + +#include +#include + +namespace Utilities +{ + //=========================================================================== + // Angle representation conversion helpers + //=========================================================================== + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + static cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles) + { + cv::Matx33d rotation_matrix; + + double s1 = sin(eulerAngles[0]); + double s2 = sin(eulerAngles[1]); + double s3 = sin(eulerAngles[2]); + + double c1 = cos(eulerAngles[0]); + double c2 = cos(eulerAngles[1]); + double c3 = cos(eulerAngles[2]); + + rotation_matrix(0, 0) = c2 * c3; + rotation_matrix(0, 1) = -c2 *s3; + rotation_matrix(0, 2) = s2; + rotation_matrix(1, 0) = c1 * s3 + c3 * s1 * s2; + rotation_matrix(1, 1) = c1 * c3 - s1 * s2 * s3; + rotation_matrix(1, 2) = -c2 * s1; + rotation_matrix(2, 0) = s1 * s3 - c1 * c3 * s2; + rotation_matrix(2, 1) = c3 * s1 + c1 * s2 * s3; + rotation_matrix(2, 2) = c1 * c2; + + return rotation_matrix; + } + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + static cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix) + { + double q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0; + double q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0*q0); + double q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0*q0); + double q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0*q0); + + double t1 = 2.0 * (q0*q2 + q1*q3); + + double yaw = asin(2.0 * (q0*q2 + q1*q3)); + double pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3); + double roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3); + + return cv::Vec3d(pitch, yaw, roll); + } + + static cv::Vec3d Euler2AxisAngle(const cv::Vec3d& euler) + { + cv::Matx33d rotMatrix = Euler2RotationMatrix(euler); + cv::Vec3d axis_angle; + cv::Rodrigues(rotMatrix, axis_angle); + return axis_angle; + } + + static cv::Vec3d AxisAngle2Euler(const cv::Vec3d& axis_angle) + { + cv::Matx33d rotation_matrix; + cv::Rodrigues(axis_angle, rotation_matrix); + return RotationMatrix2Euler(rotation_matrix); + } + + static cv::Matx33d AxisAngle2RotationMatrix(const cv::Vec3d& axis_angle) + { + cv::Matx33d rotation_matrix; + cv::Rodrigues(axis_angle, rotation_matrix); + return rotation_matrix; + } + + static cv::Vec3d RotationMatrix2AxisAngle(const cv::Matx33d& rotation_matrix) + { + cv::Vec3d axis_angle; + cv::Rodrigues(rotation_matrix, axis_angle); + return axis_angle; + } + + // Generally useful 3D functions + static void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy) + { + dest = cv::Mat_(mesh.rows, 2, 0.0); + + int num_points = mesh.rows; + + double X, Y, Z; + + cv::Mat_::const_iterator mData = mesh.begin(); + cv::Mat_::iterator projected = dest.begin(); + + for (int i = 0; i < num_points; i++) + { + // Get the points + X = *(mData++); + Y = *(mData++); + Z = *(mData++); + + double x; + double y; + + // if depth is 0 the projection is different + if (Z != 0) + { + x = ((X * fx / Z) + cx); + y = ((Y * fy / Z) + cy); + } + else + { + x = X; + y = Y; + } + + // Project and store in dest matrix + (*projected++) = x; + (*projected++) = y; + } + + } + +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h new file mode 100644 index 0000000..97f7629 --- /dev/null +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -0,0 +1,140 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __SEQUENCE_CAPTURE_h_ +#define __SEQUENCE_CAPTURE_h_ + +// System includes +#include +#include +#include + +// OpenCV includes +#include +#include + +namespace Utilities +{ + + //=========================================================================== + /** + A class for capturing sequences from video, webcam, and image directories + */ + class SequenceCapture { + + public: + + // Default constructor + SequenceCapture() {}; + + // Destructor + ~SequenceCapture(); + + // Opening based on command line arguments + bool Open(std::vector& arguments); + + // Direct opening + + // Webcam + bool OpenWebcam(int device_id, int image_width = 640, int image_height = 480, float fx = -1, float fy = -1, float cx = -1, float cy = -1); + + // Image sequence in the directory + bool OpenImageSequence(std::string directory, float fx = -1, float fy = -1, float cx = -1, float cy = -1); + + // Video file + bool OpenVideoFile(std::string video_file, float fx = -1, float fy = -1, float cx = -1, float cy = -1); + + bool IsWebcam() { return is_webcam; } + + // Getting the next frame + cv::Mat GetNextFrame(); + + // Getting the most recent grayscale frame (need to call GetNextFrame first) + cv::Mat_ GetGrayFrame(); + + // Parameters describing the sequence and it's progress + double GetProgress(); + + bool IsOpened(); + + int frame_width; + int frame_height; + + float fx, fy, cx, cy; + + double fps; + + double time_stamp; + + // Name of the video file, image directory, or the webcam + std::string name; + + // Allows to differentiate if failed because no input specified or if failed to open a specified input + bool no_input_specified; + + + private: + + // Blocking copy and move, as it doesn't make sense to have several readers pointed at the same source, and this would cause issues, especially with webcams + SequenceCapture & operator= (const SequenceCapture& other); + SequenceCapture & operator= (const SequenceCapture&& other); + SequenceCapture(const SequenceCapture&& other); + SequenceCapture(const SequenceCapture& other); + + // Used for capturing webcam and video + cv::VideoCapture capture; + + // Storing the latest captures + cv::Mat latest_frame; + cv::Mat_ latest_gray_frame; + + // Keeping track of frame number and the files in the image sequence + size_t frame_num; + std::vector image_files; + + // Length of video allowing to assess progress + size_t vid_length; + + // If using a webcam, helps to keep track of time + int64 start_time; + + // Keeping track if we are opening a video, webcam or image sequence + bool is_webcam; + bool is_image_seq; + + void SetCameraIntrinsics(float fx, float fy, float cx, float cy); + + + }; +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/include/VisualizationUtils.h b/lib/local/Utilities/include/VisualizationUtils.h new file mode 100644 index 0000000..518610f --- /dev/null +++ b/lib/local/Utilities/include/VisualizationUtils.h @@ -0,0 +1,76 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __VISUALIZATION_UTILS_h_ +#define __VISUALIZATION_UTILS_h_ + +#include + +#include +#include + +namespace Utilities +{ + + // TODO draw AU results + + // Drawing a bounding box around the face in an image + void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy); + void DrawBox(const std::vector>& lines, cv::Mat image, cv::Scalar color, int thickness); + + // Computing a bounding box to be drawn + std::vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); + + void Visualise_FHOG(const cv::Mat_& descriptor, int num_rows, int num_cols, cv::Mat& visualisation); + + class FpsTracker + { + public: + + double history_length; + + void AddFrame(); + + double GetFPS(); + + FpsTracker(); + + private: + std::queue frame_times; + + void DiscardOldFrames(); + + }; + +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/include/Visualizer.h b/lib/local/Utilities/include/Visualizer.h new file mode 100644 index 0000000..22ebaa6 --- /dev/null +++ b/lib/local/Utilities/include/Visualizer.h @@ -0,0 +1,107 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __VISUALIZER_h_ +#define __VISUALIZER_h_ + +// System includes +#include + +// OpenCV includes +#include +#include + +namespace Utilities +{ + + //=========================================================================== + /** + A class for recording data processed by OpenFace (facial landmarks, head pose, facial action units, aligned face, HOG features, and tracked video + */ + class Visualizer { + + public: + + // The constructor for the visualizer that specifies what to visualize + Visualizer(std::vector arguments); + Visualizer(bool vis_track, bool vis_hog, bool vis_align); + + // Adding observations to the visualizer + + // Pose related observations + void SetImage(const cv::Mat& canvas, float fx, float fy, float cx, float cy); + + // All observations relevant to facial landmarks (optional visibilities parameter to not display all landmarks) + void SetObservationLandmarks(const cv::Mat_& landmarks_2D, double confidence, bool success, const cv::Mat_& visibilities = cv::Mat_()); + + // Pose related observations + void SetObservationPose(const cv::Vec6d& pose, double confidence); + + // Gaze related observations + void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector& eye_landmarks, const std::vector& eye_landmarks3d, double confidence); + + // Face alignment related observations + void SetObservationFaceAlign(const cv::Mat& aligned_face); + + // HOG feature related observations + void SetObservationHOG(const cv::Mat_& hog_descriptor, int num_cols, int num_rows); + + void SetFps(double fps); + + // Return key-press that could have resulted in the open windows + char ShowObservation(); + + cv::Mat GetVisImage(); + + // Keeping track of what we're visualizing + bool vis_track; + bool vis_hog; + bool vis_align; + + // Can be adjusted to show less confident frames + double visualisation_boundary = 0.4; + + private: + + // Temporary variables for visualization + cv::Mat captured_image; // out canvas + cv::Mat tracked_image; + cv::Mat hog_image; + cv::Mat aligned_face_image; + + // Useful for drawing 3d + float fx, fy, cx, cy; + + }; +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/src/ImageCapture.cpp b/lib/local/Utilities/src/ImageCapture.cpp new file mode 100644 index 0000000..0ec7154 --- /dev/null +++ b/lib/local/Utilities/src/ImageCapture.cpp @@ -0,0 +1,421 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "ImageCapture.h" +#include "ImageManipulationHelpers.h" +#include + +// Boost includes +#include +#include +#include + +// OpenCV includes +#include + +using namespace Utilities; + +#define INFO_STREAM( stream ) \ +std::cout << stream << std::endl + +#define WARN_STREAM( stream ) \ +std::cout << "Warning: " << stream << std::endl + +#define ERROR_STREAM( stream ) \ +std::cout << "Error: " << stream << std::endl + +bool ImageCapture::Open(std::vector& arguments) +{ + + // Consuming the input arguments + bool* valid = new bool[arguments.size()]; + + for (size_t i = 0; i < arguments.size(); ++i) + { + valid[i] = true; + } + + // Some default values + std::string input_root = ""; + fx = -1; fy = -1; cx = -1; cy = -1; + frame_num = 0; + + std::string separator = std::string(1, boost::filesystem::path::preferred_separator); + + // First check if there is a root argument (so that videos and input directories could be defined more easily) + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-root") == 0) + { + input_root = arguments[i + 1] + separator; + i++; + } + if (arguments[i].compare("-inroot") == 0) + { + input_root = arguments[i + 1] + separator; + i++; + } + } + + std::string input_directory; + std::string bbox_directory; + + bool directory_found = false; + has_bounding_boxes = false; + + std::vector input_image_files; + + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-f") == 0) + { + input_image_files.push_back(input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-fdir") == 0) + { + if (directory_found) + { + WARN_STREAM("Input directory already found, using the first one:" + input_directory); + } + else + { + input_directory = (input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + directory_found = true; + } + } + else if (arguments[i].compare("-bboxdir") == 0) + { + bbox_directory = (input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + has_bounding_boxes = true; + i++; + } + else if (arguments[i].compare("-fx") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> fx; + i++; + } + else if (arguments[i].compare("-fy") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> fy; + i++; + } + else if (arguments[i].compare("-cx") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> cx; + i++; + } + else if (arguments[i].compare("-cy") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> cy; + i++; + } + } + + for (int i = (int)arguments.size() - 1; i >= 0; --i) + { + if (!valid[i]) + { + arguments.erase(arguments.begin() + i); + } + } + + // Based on what was read in open the sequence + if (!input_image_files.empty()) + { + return OpenImageFiles(input_image_files, fx, fy, cx, cy); + } + if (!input_directory.empty()) + { + return OpenDirectory(input_directory, bbox_directory, fx, fy, cx, cy); + } + + // If no input found return false and set a flag for it + no_input_specified = true; + + return false; +} + +bool ImageCapture::OpenImageFiles(const std::vector& image_files, float fx, float fy, float cx, float cy) +{ + + no_input_specified = false; + + latest_frame = cv::Mat(); + latest_gray_frame = cv::Mat(); + this->image_files = image_files; + + // Allow for setting the camera intrinsics, but have to be the same ones for every image + if (fx != -1 && fy != -1 ) + { + image_focal_length_set = true; + this->fx = fx; + this->fy = fy; + + } + else + { + image_focal_length_set = false; + } + + if (cx != -1 && cy != -1) + { + this->cx = cx; + this->cy = cy; + image_optical_center_set = true; + } + else + { + image_optical_center_set = false; + } + + return true; + +} + +bool ImageCapture::OpenDirectory(std::string directory, std::string bbox_directory, float fx, float fy, float cx, float cy) +{ + INFO_STREAM("Attempting to read from directory: " << directory); + + no_input_specified = false; + + image_files.clear(); + + boost::filesystem::path image_directory(directory); + std::vector file_in_directory; + copy(boost::filesystem::directory_iterator(image_directory), boost::filesystem::directory_iterator(), back_inserter(file_in_directory)); + + // Sort the images in the directory first + sort(file_in_directory.begin(), file_in_directory.end()); + + std::vector curr_dir_files; + + for (std::vector::const_iterator file_iterator(file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator) + { + // Possible image extension .jpg and .png + if (file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".jpeg") == 0 || file_iterator->extension().string().compare(".png") == 0 || file_iterator->extension().string().compare(".bmp") == 0) + { + curr_dir_files.push_back(file_iterator->string()); + + // If bounding box directory is specified, read the bounding boxes from it + if (!bbox_directory.empty()) + { + boost::filesystem::path current_file = *file_iterator; + boost::filesystem::path bbox_file = current_file.replace_extension("txt"); + + // If there is a bounding box file push it to the list of bounding boxes + if (boost::filesystem::exists(bbox_file)) + { + std::ifstream in_bbox(bbox_file.string().c_str(), std::ios_base::in); + + std::vector > bboxes_image; + + // Keep reading bounding boxes from a file, stop if empty line or + while (!in_bbox.eof()) + { + std::string bbox_string; + std::getline(in_bbox, bbox_string); + + if (bbox_string.empty()) + continue; + + std::stringstream ss(bbox_string); + + double min_x, min_y, max_x, max_y; + + ss >> min_x >> min_y >> max_x >> max_y; + bboxes_image.push_back(cv::Rect_(min_x, min_y, max_x - min_x, max_y - min_y)); + } + in_bbox.close(); + + bounding_boxes.push_back(bboxes_image); + } + else + { + ERROR_STREAM("Could not find the corresponding bounding box for file:" + file_iterator->string()); + exit(1); + } + } + } + } + + image_files = curr_dir_files; + + if (image_files.empty()) + { + std::cout << "No images found in the directory: " << directory << std::endl; + return false; + } + + // Allow for setting the camera intrinsics, but have to be the same ones for every image + if (fx != -1 && fy != -1) + { + image_focal_length_set = true; + this->fx = fx; + this->fy = fy; + + } + else + { + image_focal_length_set = false; + } + + if (cx != -1 && cy != -1) + { + this->cx = cx; + this->cy = cy; + image_optical_center_set = true; + } + else + { + image_optical_center_set = false; + } + + return true; + +} + +void ImageCapture::SetCameraIntrinsics(float fx, float fy, float cx, float cy) +{ + // If optical centers are not defined just use center of image + if (cx == -1) + { + this->cx = this->image_width / 2.0f; + this->cy = this->image_height / 2.0f; + } + else + { + this->cx = cx; + this->cy = cy; + } + // Use a rough guess-timate of focal length + if (fx == -1) + { + this->fx = 500.0f * (this->image_width / 640.0f); + this->fy = 500.0f * (this->image_height / 480.0f); + + this->fx = (this->fx + this->fy) / 2.0f; + this->fy = this->fx; + } + else + { + this->fx = fx; + this->fy = fy; + } +} + +// Returns a read image in 3 channel RGB format, also prepares a grayscale frame if needed +cv::Mat ImageCapture::GetNextImage() +{ + if (image_files.empty() || frame_num >= image_files.size()) + { + // Indicate lack of success by returning an empty image + latest_frame = cv::Mat(); + return latest_frame; + } + + // Load the image as an 8 bit RGB + latest_frame = cv::imread(image_files[frame_num], CV_LOAD_IMAGE_COLOR); + + if (latest_frame.empty()) + { + ERROR_STREAM("Could not open the image: " + image_files[frame_num]); + exit(1); + } + + image_height = latest_frame.size().height; + image_width = latest_frame.size().width; + + // Reset the intrinsics for every image if they are not set globally + float _fx = -1; + float _fy = -1; + + if (image_focal_length_set) + { + _fx = fx; + _fy = fy; + } + + float _cx = -1; + float _cy = -1; + + if (image_optical_center_set) + { + _cx = cx; + _cy = cy; + } + + SetCameraIntrinsics(_fx, _fy, _cx, _cy); + + // Set the grayscale frame + ConvertToGrayscale_8bit(latest_frame, latest_gray_frame); + + this->name = image_files[frame_num]; + + frame_num++; + + return latest_frame; +} + +std::vector > ImageCapture::GetBoundingBoxes() +{ + if (!bounding_boxes.empty()) + { + return bounding_boxes[frame_num - 1]; + } + else + { + return std::vector >(); + } +} + +double ImageCapture::GetProgress() +{ + return (double)frame_num / (double)image_files.size(); +} + +cv::Mat_ ImageCapture::GetGrayFrame() +{ + return latest_gray_frame; +} \ No newline at end of file diff --git a/lib/local/Utilities/src/RecorderCSV.cpp b/lib/local/Utilities/src/RecorderCSV.cpp new file mode 100644 index 0000000..3754261 --- /dev/null +++ b/lib/local/Utilities/src/RecorderCSV.cpp @@ -0,0 +1,347 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "RecorderCSV.h" + +// For sorting +#include + +// For standard out +#include +#include +#include + +using namespace Utilities; + +// Default constructor initializes the variables +RecorderCSV::RecorderCSV():output_file(){}; + +// Making sure full stop is used for decimal point separation +struct fullstop : std::numpunct { + char do_decimal_point() const { return '.'; } +}; + +// Opening the file and preparing the header for it +bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, + int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector& au_names_class, const std::vector& au_names_reg) +{ + + output_file.open(output_file_name, std::ios_base::out); + output_file.imbue(std::locale(output_file.getloc(), new fullstop)); + + if (!output_file.is_open()) + return false; + + this->is_sequence = is_sequence; + + // Set up what we are recording + this->output_2D_landmarks = output_2D_landmarks; + this->output_3D_landmarks = output_3D_landmarks; + this->output_AUs = output_AUs; + this->output_gaze = output_gaze; + this->output_model_params = output_model_params; + this->output_pose = output_pose; + + this->au_names_class = au_names_class; + this->au_names_reg = au_names_reg; + + // Different headers if we are writing out the results on a sequence or an individual image + if(this->is_sequence) + { + output_file << "frame, timestamp, confidence, success"; + } + else + { + output_file << "face, confidence"; + } + + if (output_gaze) + { + output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y"; + + for (int i = 0; i < num_eye_landmarks; ++i) + { + output_file << ", eye_lmk_x_" << i; + } + for (int i = 0; i < num_eye_landmarks; ++i) + { + output_file << ", eye_lmk_y_" << i; + } + + for (int i = 0; i < num_eye_landmarks; ++i) + { + output_file << ", eye_lmk_X_" << i; + } + for (int i = 0; i < num_eye_landmarks; ++i) + { + output_file << ", eye_lmk_Y_" << i; + } + for (int i = 0; i < num_eye_landmarks; ++i) + { + output_file << ", eye_lmk_Z_" << i; + } + } + + if (output_pose) + { + output_file << ", pose_Tx, pose_Ty, pose_Tz, pose_Rx, pose_Ry, pose_Rz"; + } + + if (output_2D_landmarks) + { + for (int i = 0; i < num_face_landmarks; ++i) + { + output_file << ", x_" << i; + } + for (int i = 0; i < num_face_landmarks; ++i) + { + output_file << ", y_" << i; + } + } + + if (output_3D_landmarks) + { + for (int i = 0; i < num_face_landmarks; ++i) + { + output_file << ", X_" << i; + } + for (int i = 0; i < num_face_landmarks; ++i) + { + output_file << ", Y_" << i; + } + for (int i = 0; i < num_face_landmarks; ++i) + { + output_file << ", Z_" << i; + } + } + + // Outputting model parameters (rigid and non-rigid), the first parameters are the 6 rigid shape parameters, they are followed by the non rigid shape parameters + if (output_model_params) + { + output_file << ", p_scale, p_rx, p_ry, p_rz, p_tx, p_ty"; + for (int i = 0; i < num_model_modes; ++i) + { + output_file << ", p_" << i; + } + } + + if (output_AUs) + { + std::sort(this->au_names_reg.begin(), this->au_names_reg.end()); + for (std::string reg_name : this->au_names_reg) + { + output_file << ", " << reg_name << "_r"; + } + + std::sort(this->au_names_class.begin(), this->au_names_class.end()); + for (std::string class_name : this->au_names_class) + { + output_file << ", " << class_name << "_c"; + } + } + + output_file << std::endl; + + return true; + +} + +void RecorderCSV::WriteLine(int observation_count, double time_stamp, bool landmark_detection_success, double landmark_confidence, + const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, const cv::Mat_& pdm_model_params, const cv::Vec6d& rigid_shape_params, cv::Vec6d& pose_estimate, + const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, + const std::vector >& au_intensities, const std::vector >& au_occurences) +{ + + if (!output_file.is_open()) + { + std::cout << "The output CSV file is not open, exiting" << std::endl; + exit(1); + } + + // Making sure fixed and not scientific notation is used + output_file << std::fixed; + output_file << std::noshowpoint; + if(is_sequence) + { + output_file << std::setprecision(3); + output_file << observation_count << ", " << time_stamp; + output_file << std::setprecision(2); + output_file << ", " << landmark_confidence; + output_file << std::setprecision(0); + output_file << ", " << landmark_detection_success; + } + else + { + output_file << std::setprecision(3); + output_file << observation_count << ", " << landmark_confidence; + } + // Output the estimated gaze + if (output_gaze) + { + output_file << std::setprecision(6); + output_file << ", " << gazeDirection0.x << ", " << gazeDirection0.y << ", " << gazeDirection0.z + << ", " << gazeDirection1.x << ", " << gazeDirection1.y << ", " << gazeDirection1.z; + + // Output gaze angle (same format as head pose angle) + output_file << std::setprecision(3); + output_file << ", " << gaze_angle[0] << ", " << gaze_angle[1]; + + // Output the 2D eye landmarks + output_file << std::setprecision(1); + for (auto eye_lmk : eye_landmarks2d) + { + output_file << ", " << eye_lmk.x; + } + + for (auto eye_lmk : eye_landmarks2d) + { + output_file << ", " << eye_lmk.y; + } + + // Output the 3D eye landmarks + for (auto eye_lmk : eye_landmarks3d) + { + output_file << ", " << eye_lmk.x; + } + + for (auto eye_lmk : eye_landmarks3d) + { + output_file << ", " << eye_lmk.y; + } + + for (auto eye_lmk : eye_landmarks3d) + { + output_file << ", " << eye_lmk.z; + } + } + + // Output the estimated head pose + if (output_pose) + { + output_file << std::setprecision(1); + output_file << ", " << pose_estimate[0] << ", " << pose_estimate[1] << ", " << pose_estimate[2]; + output_file << std::setprecision(3); + output_file << ", " << pose_estimate[3] << ", " << pose_estimate[4] << ", " << pose_estimate[5]; + } + + // Output the detected 2D facial landmarks + if (output_2D_landmarks) + { + output_file.precision(1); + // Output the 2D eye landmarks + for (auto lmk : landmarks_2D) + { + output_file << ", " << lmk; + } + } + + // Output the detected 3D facial landmarks + if (output_3D_landmarks) + { + output_file.precision(1); + // Output the 2D eye landmarks + for (auto lmk : landmarks_3D) + { + output_file << ", " << lmk; + } + } + + if (output_model_params) + { + output_file.precision(3); + for (int i = 0; i < 6; ++i) + { + output_file << ", " << rigid_shape_params[i]; + } + // Output the non_rigid shape parameters + for (auto lmk : pdm_model_params) + { + output_file << ", " << lmk; + } + } + + if (output_AUs) + { + + // write out ar the correct index + output_file.precision(2); + for (std::string au_name : au_names_reg) + { + for (auto au_reg : au_intensities) + { + if (au_name.compare(au_reg.first) == 0) + { + output_file << ", " << au_reg.second; + break; + } + } + } + + if (au_intensities.size() == 0) + { + for (size_t p = 0; p < au_names_reg.size(); ++p) + { + output_file << ", 0"; + } + } + + output_file.precision(1); + // write out ar the correct index + for (std::string au_name : au_names_class) + { + for (auto au_class : au_occurences) + { + if (au_name.compare(au_class.first) == 0) + { + output_file << ", " << au_class.second; + break; + } + } + } + + if (au_occurences.size() == 0) + { + for (size_t p = 0; p < au_names_class.size(); ++p) + { + output_file << ", 0"; + } + } + } + output_file << std::endl; +} + +// Closing the file and cleaning up +void RecorderCSV::Close() +{ + output_file.close(); +} diff --git a/lib/local/Utilities/src/RecorderHOG.cpp b/lib/local/Utilities/src/RecorderHOG.cpp new file mode 100644 index 0000000..eb469ec --- /dev/null +++ b/lib/local/Utilities/src/RecorderHOG.cpp @@ -0,0 +1,95 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "RecorderHOG.h" + +#include + +using namespace Utilities; + +// Default constructor initializes the variables +RecorderHOG::RecorderHOG() :hog_file() {}; + +// Opening the file and preparing the header for it +bool RecorderHOG::Open(std::string output_file_name) +{ + hog_file.open(output_file_name, std::ios_base::out | std::ios_base::binary); + + return hog_file.is_open(); +} + +void RecorderHOG::Close() +{ + hog_file.close(); +} + +void RecorderHOG::Write() +{ + hog_file.write((char*)(&num_cols), 4); + hog_file.write((char*)(&num_rows), 4); + hog_file.write((char*)(&num_channels), 4); + + // Not the best way to store a bool, but will be much easier to read it + float good_frame_float; + if (good_frame) + good_frame_float = 1; + else + good_frame_float = -1; + + hog_file.write((char*)(&good_frame_float), 4); + + cv::MatConstIterator_ descriptor_it = hog_descriptor.begin(); + + for (int y = 0; y < num_cols; ++y) + { + for (int x = 0; x < num_rows; ++x) + { + for (unsigned int o = 0; o < 31; ++o) + { + + float hog_data = (float)(*descriptor_it++); + hog_file.write((char*)&hog_data, 4); + } + } + } +} + +// Writing to a HOG file +void RecorderHOG::SetObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels) +{ + this->num_cols = num_cols; + this->num_rows = num_rows; + this->num_channels = num_channels; + this->hog_descriptor = hog_descriptor; + this->good_frame = good_frame; +} \ No newline at end of file diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp new file mode 100644 index 0000000..d8ff222 --- /dev/null +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -0,0 +1,408 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "RecorderOpenFace.h" + +// For sorting +#include + +// File manipulation +#include +#include +#include + +// Boost includes for file system manipulation +#include +#include +#include + +using namespace boost::filesystem; + +using namespace Utilities; + +#define WARN_STREAM( stream ) \ +std::cout << "Warning: " << stream << std::endl + +void CreateDirectory(std::string output_path) +{ + + // Creating the right directory structure + auto p = path(output_path); + + if (!boost::filesystem::exists(p)) + { + bool success = boost::filesystem::create_directories(p); + + if (!success) + { + std::cout << "ERROR: failed to create output directory:" << p.string() << ", do you have permission to create directory" << std::endl; + exit(1); + } + } +} + +RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::vector& arguments):video_writer(), params(parameters) +{ + + // From the filename, strip out the name without directory and extension + if (boost::filesystem::is_directory(in_filename)) + { + filename = boost::filesystem::canonical(boost::filesystem::path(in_filename)).filename().string(); + } + else + { + filename = boost::filesystem::path(in_filename).filename().replace_extension("").string(); + } + + // Consuming the input arguments + bool* valid = new bool[arguments.size()]; + + for (size_t i = 0; i < arguments.size(); ++i) + { + valid[i] = true; + } + + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-out_dir") == 0) + { + record_root = arguments[i + 1]; + } + } + + // Determine output directory + bool output_found = false; + for (size_t i = 0; i < arguments.size(); ++i) + { + if (!output_found && arguments[i].compare("-of") == 0) + { + record_root = (boost::filesystem::path(record_root) / boost::filesystem::path(arguments[i + 1])).remove_filename().string(); + filename = path(boost::filesystem::path(arguments[i + 1])).replace_extension("").filename().string(); + valid[i] = false; + valid[i + 1] = false; + i++; + output_found = true; + } + } + + // If recording directory not set, record to default location + if (record_root.empty()) + record_root = default_record_directory; + + for (int i = (int)arguments.size() - 1; i >= 0; --i) + { + if (!valid[i]) + { + arguments.erase(arguments.begin() + i); + } + } + + // Construct the directories required for the output + CreateDirectory(record_root); + + // Create the filename for the general output file that contains all of the meta information about the recording + path of_det_name(filename); + of_det_name = path(record_root) / path(filename + "_of_details.txt"); + + // Write in the of file what we are outputing what is the input etc. + metadata_file.open(of_det_name.string(), std::ios_base::out); + if (!metadata_file.is_open()) + { + cout << "ERROR: could not open the output file:" << of_det_name << ", either the path of the output directory is wrong or you do not have the permissions to write to it" << endl; + exit(1); + } + + // Populate relative and full path names in the meta file, unless it is a webcam + if(!params.isFromWebcam()) + { + string input_filename_relative = in_filename; + string input_filename_full = in_filename; + if (!boost::filesystem::path(input_filename_full).is_absolute()) + { + input_filename_full = boost::filesystem::canonical(input_filename_relative).string(); + } + metadata_file << "Input:" << input_filename_relative << endl; + metadata_file << "Input full path:" << input_filename_full << endl; + } + else + { + // Populate the metadata file + metadata_file << "Input:webcam" << endl; + } + + metadata_file << "Camera parameters:" << parameters.getFx() << "," << parameters.getFy() << "," << parameters.getCx() << "," << parameters.getCy() << endl; + + // Create the required individual recorders, CSV, HOG, aligned, video + csv_filename = filename + ".csv"; + + // Consruct HOG recorder here + if(params.outputHOG()) + { + // Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory + std::string hog_filename = filename + ".hog"; + metadata_file << "Output HOG:" << hog_filename << endl; + hog_filename = (path(record_root) / hog_filename).string(); + hog_recorder.Open(hog_filename); + } + + // saving the videos + if (params.outputTracked()) + { + if(parameters.isSequence()) + { + // Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory + this->media_filename = filename + ".avi"; + metadata_file << "Output video:" << this->media_filename << endl; + this->media_filename = (path(record_root) / this->media_filename).string(); + } + else + { + this->media_filename = filename + ".jpg"; + metadata_file << "Output image:" << this->media_filename << endl; + this->media_filename = (path(record_root) / this->media_filename).string(); + } + } + + // Prepare image recording + if (params.outputAlignedFaces()) + { + aligned_output_directory = filename + "_aligned"; + metadata_file << "Output aligned directory:" << this->aligned_output_directory << endl; + this->aligned_output_directory = (path(record_root) / this->aligned_output_directory).string(); + CreateDirectory(aligned_output_directory); + } + + observation_count = 0; + +} + +void RecorderOpenFace::SetObservationFaceAlign(const cv::Mat& aligned_face) +{ + this->aligned_face = aligned_face; +} + +void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) +{ + if (params.outputTracked()) + { + // Initialize the video writer if it has not been opened yet + if(params.isSequence() && !video_writer.isOpened()) + { + std::string output_codec = params.outputCodec(); + try + { + video_writer.open(media_filename, CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), params.outputFps(), vis_track.size(), true); + + if (!video_writer.isOpened()) + { + WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN."); + } + } + catch (cv::Exception e) + { + WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); + } + + } + + vis_to_out = vis_track; + + } + +} + +void RecorderOpenFace::WriteObservation() +{ + observation_count++; + + // Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers) + + if(observation_count == 1) + { + // As we are writing out the header, work out some things like number of landmarks, names of AUs etc. + int num_face_landmarks = landmarks_2D.rows / 2; + int num_eye_landmarks = (int)eye_landmarks2D.size(); + int num_model_modes = pdm_params_local.rows; + + std::vector au_names_class; + for (auto au : au_occurences) + { + au_names_class.push_back(au.first); + } + + std::sort(au_names_class.begin(), au_names_class.end()); + + std::vector au_names_reg; + for (auto au : au_intensities) + { + au_names_reg.push_back(au.first); + } + + std::sort(au_names_reg.begin(), au_names_reg.end()); + + metadata_file << "Output csv:" << csv_filename << endl; + metadata_file << "Gaze: " << params.outputGaze() << endl; + metadata_file << "AUs: " << params.outputAUs() << endl; + metadata_file << "Landmarks 2D: " << params.output2DLandmarks() << endl; + metadata_file << "Landmarks 3D: " << params.output3DLandmarks() << endl; + metadata_file << "Pose: " << params.outputPose() << endl; + metadata_file << "Shape parameters: " << params.outputPDMParams() << endl; + + csv_filename = (path(record_root) / csv_filename).string(); + csv_recorder.Open(csv_filename, params.isSequence(), params.output2DLandmarks(), params.output3DLandmarks(), params.outputPDMParams(), params.outputPose(), + params.outputAUs(), params.outputGaze(), num_face_landmarks, num_model_modes, num_eye_landmarks, au_names_class, au_names_reg); + } + + this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success, + landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose, + gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks2D, eye_landmarks3D, au_intensities, au_occurences); + + if(params.outputHOG()) + { + this->hog_recorder.Write(); + } + + // Write aligned faces + if (params.outputAlignedFaces()) + { + char name[100]; + + // Filename is based on frame number + if(params.isSequence()) + std::sprintf(name, "frame_det_%06d.bmp", observation_count); + else + std::sprintf(name, "face_det_%06d.bmp", observation_count); + + // Construct the output filename + boost::filesystem::path slash("/"); + + std::string preferredSlash = slash.make_preferred().string(); + + string out_file = aligned_output_directory + preferredSlash + string(name); + bool write_success = cv::imwrite(out_file, aligned_face); + + if (!write_success) + { + WARN_STREAM("Could not output similarity aligned image image"); + } + } + + if(params.outputTracked()) + { + if (vis_to_out.empty()) + { + WARN_STREAM("Output tracked video frame is not set"); + } + + if(params.isSequence() ) + { + if(video_writer.isOpened()) + { + video_writer.write(vis_to_out); + } + } + else + { + bool out_success = cv::imwrite(media_filename, vis_to_out); + if (!out_success) + { + WARN_STREAM("Could not output tracked image"); + } + } + // Clear the output + vis_to_out = cv::Mat(); + } +} + + +void RecorderOpenFace::SetObservationHOG(bool good_frame, const cv::Mat_& hog_descriptor, int num_cols, int num_rows, int num_channels) +{ + this->hog_recorder.SetObservationHOG(good_frame, hog_descriptor, num_cols, num_rows, num_channels); +} + +void RecorderOpenFace::SetObservationTimestamp(double timestamp) +{ + this->timestamp = timestamp; +} + +void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_& landmarks_2D, const cv::Mat_& landmarks_3D, + const cv::Vec6d& pdm_params_global, const cv::Mat_& pdm_params_local, double confidence, bool success) +{ + this->landmarks_2D = landmarks_2D; + this->landmarks_3D = landmarks_3D; + this->pdm_params_global = pdm_params_global; + this->pdm_params_local = pdm_params_local; + this->landmark_detection_confidence = confidence; + this->landmark_detection_success = success; + +} + +void RecorderOpenFace::SetObservationPose(const cv::Vec6d& pose) +{ + this->head_pose = pose; +} + +void RecorderOpenFace::SetObservationActionUnits(const std::vector >& au_intensities, + const std::vector >& au_occurences) +{ + this->au_intensities = au_intensities; + this->au_occurences = au_occurences; +} + +void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, + const cv::Vec2d& gaze_angle, const std::vector& eye_landmarks2D, const std::vector& eye_landmarks3D) +{ + this->gaze_direction0 = gaze_direction0; + this->gaze_direction1 = gaze_direction1; + this->gaze_angle = gaze_angle; + this->eye_landmarks2D = eye_landmarks2D; + this->eye_landmarks3D = eye_landmarks3D; +} + +RecorderOpenFace::~RecorderOpenFace() +{ + this->Close(); +} + + +void RecorderOpenFace::Close() +{ + hog_recorder.Close(); + csv_recorder.Close(); + video_writer.release(); + metadata_file.close(); + +} + + + diff --git a/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp new file mode 100644 index 0000000..20b58df --- /dev/null +++ b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp @@ -0,0 +1,140 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "RecorderOpenFaceParameters.h" + +using namespace std; + +using namespace Utilities; + +RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector &arguments, bool sequence, bool from_webcam, float fx, float fy, float cx, float cy, double fps_vid_out) +{ + + string separator = string(1, boost::filesystem::path::preferred_separator); + + this->is_sequence = sequence; + this->is_from_webcam = from_webcam; + this->fx = fx; + this->fy = fy; + this->cx = cx; + this->cy = cy; + + if(fps_vid_out > 0) + { + this->fps_vid_out = fps_vid_out; + } + else + { + this->fps_vid_out = 30; // If an illegal value for fps provided, default to 30 + } + // Default output code + this->output_codec = "DIVX"; + + bool output_set = false; + + output_2D_landmarks = false; + output_3D_landmarks = false; + output_model_params = false; + output_pose = false; + output_AUs = false; + output_gaze = false; + output_hog = false; + output_tracked = false; + output_aligned_faces = false; + + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-simalign") == 0) + { + output_aligned_faces = true; + output_set = true; + } + else if (arguments[i].compare("-hogalign") == 0) + { + output_hog = true; + output_set = true; + } + else if (arguments[i].compare("-2Dfp") == 0) + { + output_2D_landmarks = true; + output_set = true; + } + else if (arguments[i].compare("-3Dfp") == 0) + { + output_3D_landmarks = true; + output_set = true; + } + else if (arguments[i].compare("-pdmparams") == 0) + { + output_model_params = true; + output_set = true; + } + else if (arguments[i].compare("-pose") == 0) + { + output_pose = true; + output_set = true; + } + else if (arguments[i].compare("-aus") == 0) + { + output_AUs = true; + output_set = true; + } + else if (arguments[i].compare("-gaze") == 0) + { + output_gaze = true; + output_set = true; + } + else if (arguments[i].compare("-tracked") == 0) + { + output_tracked = true; + output_set = true; + } + } + + // Output everything if nothing has been set + + if (!output_set) + { + output_2D_landmarks = true; + output_3D_landmarks = true; + output_model_params = true; + output_pose = true; + output_AUs = true; + output_gaze = true; + output_hog = true; + output_tracked = true; + output_aligned_faces = true; + } + +} + diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp new file mode 100644 index 0000000..7482575 --- /dev/null +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -0,0 +1,461 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "SequenceCapture.h" +#include "ImageManipulationHelpers.h" + +#include + +// Boost includes +#include +#include +#include + +// OpenCV includes +#include + +// For timing +#include +#include + +using namespace Utilities; + +#define INFO_STREAM( stream ) \ +std::cout << stream << std::endl + +#define WARN_STREAM( stream ) \ +std::cout << "Warning: " << stream << std::endl + +#define ERROR_STREAM( stream ) \ +std::cout << "Error: " << stream << std::endl + +bool SequenceCapture::Open(std::vector& arguments) +{ + + // Consuming the input arguments + bool* valid = new bool[arguments.size()]; + + for (size_t i = 0; i < arguments.size(); ++i) + { + valid[i] = true; + } + + // Some default values + std::string input_root = ""; + fx = -1; fy = -1; cx = -1; cy = -1; + frame_num = 0; + time_stamp = 0; + + std::string separator = std::string(1, boost::filesystem::path::preferred_separator); + + // First check if there is a root argument (so that videos and input directories could be defined more easily) + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-root") == 0) + { + input_root = arguments[i + 1] + separator; + i++; + } + if (arguments[i].compare("-inroot") == 0) + { + input_root = arguments[i + 1] + separator; + i++; + } + } + + std::string input_video_file; + std::string input_sequence_directory; + int device = -1; + + bool file_found = false; + + for (size_t i = 0; i < arguments.size(); ++i) + { + if (!file_found && arguments[i].compare("-f") == 0) + { + input_video_file = (input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + file_found = true; + } + else if (!file_found && arguments[i].compare("-fdir") == 0) + { + input_sequence_directory = (input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + file_found = true; + } + else if (arguments[i].compare("-fx") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> fx; + i++; + } + else if (arguments[i].compare("-fy") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> fy; + i++; + } + else if (arguments[i].compare("-cx") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> cx; + i++; + } + else if (arguments[i].compare("-cy") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> cy; + i++; + } + else if (arguments[i].compare("-device") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> device; + valid[i] = false; + valid[i + 1] = false; + i++; + } + } + + for (int i = (int)arguments.size() - 1; i >= 0; --i) + { + if (!valid[i]) + { + arguments.erase(arguments.begin() + i); + } + } + + no_input_specified = !file_found; + + // Based on what was read in open the sequence + if (device != -1) + { + // TODO allow to specify webcam resolution + return OpenWebcam(device, 640, 480, fx, fy, cx, cy); + } + if (!input_video_file.empty()) + { + return OpenVideoFile(input_video_file, fx, fy, cx, cy); + } + if (!input_sequence_directory.empty()) + { + return OpenImageSequence(input_sequence_directory, fx, fy, cx, cy); + } + + // If no input found return false and set a flag for it + no_input_specified = true; + + return false; +} + +// Get current date/time, format is YYYY-MM-DD.HH:mm, useful for saving data from webcam +const std::string currentDateTime() +{ + + time_t rawtime; + struct tm * timeinfo; + char buffer[80]; + + time(&rawtime); + timeinfo = localtime(&rawtime); + + strftime(buffer, sizeof(buffer), "%Y-%m-%d-%H-%M", timeinfo); + + return buffer; +} + + +bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, float fx, float fy, float cx, float cy) +{ + INFO_STREAM("Attempting to read from webcam: " << device); + + no_input_specified = false; + + if (device < 0) + { + std::cout << "Specify a valid device" << std::endl; + return false; + } + + latest_frame = cv::Mat(); + latest_gray_frame = cv::Mat(); + + capture.open(device); + capture.set(CV_CAP_PROP_FRAME_WIDTH, image_width); + capture.set(CV_CAP_PROP_FRAME_HEIGHT, image_height); + + is_webcam = true; + is_image_seq = false; + + vid_length = 0; + + this->frame_width = (int)capture.get(CV_CAP_PROP_FRAME_WIDTH); + this->frame_height = (int)capture.get(CV_CAP_PROP_FRAME_HEIGHT); + + if (!capture.isOpened()) + { + std::cout << "Failed to open the webcam" << std::endl; + return false; + } + if (frame_width != image_width || frame_height != image_height) + { + std::cout << "Failed to open the webcam with desired resolution" << std::endl; + std::cout << "Defaulting to " << frame_width << "x" << frame_height << std::endl; + } + + this->fps = capture.get(CV_CAP_PROP_FPS); + + // Check if fps is nan or less than 0 + if (fps != fps || fps <= 0) + { + INFO_STREAM("FPS of the webcam cannot be determined, assuming 30"); + fps = 30; + } + + SetCameraIntrinsics(fx, fy, cx, cy); + std::string time = currentDateTime(); + this->name = "webcam_" + time; + + start_time = cv::getTickCount(); + + return true; + +} + +// Destructor that releases the capture +SequenceCapture::~SequenceCapture() +{ + if (capture.isOpened()) + capture.release(); +} + +bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, float cx, float cy) +{ + INFO_STREAM("Attempting to read from file: " << video_file); + + no_input_specified = false; + + latest_frame = cv::Mat(); + latest_gray_frame = cv::Mat(); + + capture.open(video_file); + + if (!capture.isOpened()) + { + std::cout << "Failed to open the video file at location: " << video_file << std::endl; + return false; + } + + this->fps = capture.get(CV_CAP_PROP_FPS); + + // Check if fps is nan or less than 0 + if (fps != fps || fps <= 0) + { + WARN_STREAM("FPS of the video file cannot be determined, assuming 30"); + fps = 30; + } + + is_webcam = false; + is_image_seq = false; + + this->frame_width = (int)capture.get(CV_CAP_PROP_FRAME_WIDTH); + this->frame_height = (int)capture.get(CV_CAP_PROP_FRAME_HEIGHT); + + vid_length = (int)capture.get(CV_CAP_PROP_FRAME_COUNT); + + SetCameraIntrinsics(fx, fy, cx, cy); + + this->name = video_file; + + return true; + +} + +bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float fy, float cx, float cy) +{ + INFO_STREAM("Attempting to read from directory: " << directory); + + no_input_specified = false; + + image_files.clear(); + + boost::filesystem::path image_directory(directory); + std::vector file_in_directory; + copy(boost::filesystem::directory_iterator(image_directory), boost::filesystem::directory_iterator(), back_inserter(file_in_directory)); + + // Sort the images in the directory first + sort(file_in_directory.begin(), file_in_directory.end()); + + std::vector curr_dir_files; + + for (std::vector::const_iterator file_iterator(file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator) + { + // Possible image extension .jpg and .png + if (file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".jpeg") == 0 || file_iterator->extension().string().compare(".png") == 0 || file_iterator->extension().string().compare(".bmp") == 0) + { + curr_dir_files.push_back(file_iterator->string()); + } + } + + image_files = curr_dir_files; + + if (image_files.empty()) + { + std::cout << "No images found in the directory: " << directory << std::endl; + return false; + } + + // Assume all images are same size in an image sequence + cv::Mat tmp = cv::imread(image_files[0], CV_LOAD_IMAGE_COLOR); + this->frame_height = tmp.size().height; + this->frame_width = tmp.size().width; + + SetCameraIntrinsics(fx, fy, cx, cy); + + // No fps as we have a sequence + this->fps = 0; + + this->name = directory; + + is_webcam = false; + is_image_seq = true; + vid_length = image_files.size(); + + return true; + +} + +void SequenceCapture::SetCameraIntrinsics(float fx, float fy, float cx, float cy) +{ + // If optical centers are not defined just use center of image + if (cx == -1) + { + this->cx = this->frame_width / 2.0f; + this->cy = this->frame_height / 2.0f; + } + else + { + this->cx = cx; + this->cy = cy; + } + // Use a rough guess-timate of focal length + if (fx == -1) + { + this->fx = 500.0f * (this->frame_width / 640.0f); + this->fy = 500.0f * (this->frame_height / 480.0f); + + this->fx = (this->fx + this->fy) / 2.0f; + this->fy = this->fx; + } + else + { + this->fx = fx; + this->fy = fy; + } +} + +cv::Mat SequenceCapture::GetNextFrame() +{ + + if (is_webcam || !is_image_seq) + { + + bool success = capture.read(latest_frame); + + if (!success) + { + // Indicate lack of success by returning an empty image + latest_frame = cv::Mat(); + } + + // Recording the timestamp + if (!is_webcam) + { + time_stamp = frame_num * (1.0 / fps); + } + else + { + time_stamp = (cv::getTickCount() - start_time) / cv::getTickFrequency(); + } + + } + else if (is_image_seq) + { + if (image_files.empty() || frame_num >= image_files.size()) + { + // Indicate lack of success by returning an empty image + latest_frame = cv::Mat(); + } + else + { + latest_frame = cv::imread(image_files[frame_num], CV_LOAD_IMAGE_COLOR); + } + time_stamp = 0; + } + + // Set the grayscale frame + ConvertToGrayscale_8bit(latest_frame, latest_gray_frame); + + frame_num++; + + return latest_frame; +} + +double SequenceCapture::GetProgress() +{ + if (is_webcam) + { + return -1.0; + } + else + { + return (double)frame_num / (double)vid_length; + } +} + +bool SequenceCapture::IsOpened() +{ + if (is_webcam || !is_image_seq) + return capture.isOpened(); + else + return (image_files.size() > 0 && frame_num < image_files.size()); +} + +cv::Mat_ SequenceCapture::GetGrayFrame() +{ + return latest_gray_frame; +} \ No newline at end of file diff --git a/lib/local/Utilities/src/VisualizationUtils.cpp b/lib/local/Utilities/src/VisualizationUtils.cpp new file mode 100644 index 0000000..8b9f5bc --- /dev/null +++ b/lib/local/Utilities/src/VisualizationUtils.cpp @@ -0,0 +1,191 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "VisualizationUtils.h" +#include "RotationHelpers.h" + +// For FHOG visualisation +#include +#include + +// For drawing on images +#include + +namespace Utilities +{ + + FpsTracker::FpsTracker() + { + // Keep two seconds of history + history_length = 2; + } + + void FpsTracker::AddFrame() + { + double current_time = cv::getTickCount() / cv::getTickFrequency(); + frame_times.push(current_time); + DiscardOldFrames(); + } + + double FpsTracker::GetFPS() + { + DiscardOldFrames(); + + if (frame_times.size() == 0) + return 0; + + double current_time = cv::getTickCount() / cv::getTickFrequency(); + + return ((double)frame_times.size()) / (current_time - frame_times.front()); + } + + void FpsTracker::DiscardOldFrames() + { + double current_time = cv::getTickCount() / cv::getTickFrequency(); + // Remove old history + while (frame_times.size() > 0 && (current_time - frame_times.front()) > history_length) + frame_times.pop(); + } + + void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy) + { + auto edge_lines = CalculateBox(pose, fx, fy, cx, cy); + DrawBox(edge_lines, image, color, thickness); + } + + std::vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy) + { + double boxVerts[] = { -1, 1, -1, + 1, 1, -1, + 1, 1, 1, + -1, 1, 1, + 1, -1, 1, + 1, -1, -1, + -1, -1, -1, + -1, -1, 1 }; + + std::vector> edges; + edges.push_back(std::pair(0, 1)); + edges.push_back(std::pair(1, 2)); + edges.push_back(std::pair(2, 3)); + edges.push_back(std::pair(0, 3)); + edges.push_back(std::pair(2, 4)); + edges.push_back(std::pair(1, 5)); + edges.push_back(std::pair(0, 6)); + edges.push_back(std::pair(3, 7)); + edges.push_back(std::pair(6, 5)); + edges.push_back(std::pair(5, 4)); + edges.push_back(std::pair(4, 7)); + edges.push_back(std::pair(7, 6)); + + // The size of the head is roughly 200mm x 200mm x 200mm + cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100; + + cv::Matx33d rot = Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5])); + cv::Mat_ rotBox; + + // Rotate the box + rotBox = cv::Mat(rot) * box.t(); + rotBox = rotBox.t(); + + // Move the bounding box to head position + rotBox.col(0) = rotBox.col(0) + pose[0]; + rotBox.col(1) = rotBox.col(1) + pose[1]; + rotBox.col(2) = rotBox.col(2) + pose[2]; + + // draw the lines + cv::Mat_ rotBoxProj; + Project(rotBoxProj, rotBox, fx, fy, cx, cy); + + std::vector> lines; + + for (size_t i = 0; i < edges.size(); ++i) + { + cv::Mat_ begin; + cv::Mat_ end; + + rotBoxProj.row(edges[i].first).copyTo(begin); + rotBoxProj.row(edges[i].second).copyTo(end); + + cv::Point2d p1(begin.at(0), begin.at(1)); + cv::Point2d p2(end.at(0), end.at(1)); + + lines.push_back(std::pair(p1, p2)); + + } + + return lines; + } + + void DrawBox(const std::vector>& lines, cv::Mat image, cv::Scalar color, int thickness) + { + cv::Rect image_rect(0, 0, image.cols, image.rows); + + for (size_t i = 0; i < lines.size(); ++i) + { + cv::Point2d p1 = lines.at(i).first; + cv::Point2d p2 = lines.at(i).second; + // Only draw the line if one of the points is inside the image + if (p1.inside(image_rect) || p2.inside(image_rect)) + { + cv::line(image, p1, p2, color, thickness, CV_AA); + } + + } + + } + + void Visualise_FHOG(const cv::Mat_& descriptor, int num_rows, int num_cols, cv::Mat& visualisation) + { + + // First convert to dlib format + dlib::array2d > hog(num_rows, num_cols); + + cv::MatConstIterator_ descriptor_it = descriptor.begin(); + for (int y = 0; y < num_cols; ++y) + { + for (int x = 0; x < num_rows; ++x) + { + for (unsigned int o = 0; o < 31; ++o) + { + hog[y][x](o) = *descriptor_it++; + } + } + } + + // Draw the FHOG to OpenCV format + auto fhog_vis = dlib::draw_fhog(hog); + visualisation = dlib::toMat(fhog_vis).clone(); + } + +} \ No newline at end of file diff --git a/lib/local/Utilities/src/Visualizer.cpp b/lib/local/Utilities/src/Visualizer.cpp new file mode 100644 index 0000000..75877f9 --- /dev/null +++ b/lib/local/Utilities/src/Visualizer.cpp @@ -0,0 +1,294 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltruaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltruaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltruaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "Visualizer.h" +#include "VisualizationUtils.h" +#include "RotationHelpers.h" +#include "ImageManipulationHelpers.h" + +// For drawing on images +#include + +using namespace Utilities; + +// For subpixel accuracy drawing +const int draw_shiftbits = 4; +const int draw_multiplier = 1 << 4; + +Visualizer::Visualizer(std::vector arguments) +{ + // By default not visualizing anything + this->vis_track = false; + this->vis_hog = false; + this->vis_align = false; + + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-verbose") == 0) + { + vis_track = true; + vis_align = true; + vis_hog = true; + } + else if (arguments[i].compare("-vis-align") == 0) + { + vis_align = true; + } + else if (arguments[i].compare("-vis-hog") == 0) + { + vis_hog = true; + } + else if (arguments[i].compare("-vis-track") == 0) + { + vis_track = true; + } + } + +} + +Visualizer::Visualizer(bool vis_track, bool vis_hog, bool vis_align) +{ + this->vis_track = vis_track; + this->vis_hog = vis_hog; + this->vis_align = vis_align; +} + + + +// Setting the image on which to draw +void Visualizer::SetImage(const cv::Mat& canvas, float fx, float fy, float cx, float cy) +{ + // Convert the image to 8 bit RGB + captured_image = canvas.clone(); + + this->fx = fx; + this->fy = fy; + this->cx = cx; + this->cy = cy; + + // Clearing other images + hog_image = cv::Mat(); + aligned_face_image = cv::Mat(); + +} + + +void Visualizer::SetObservationFaceAlign(const cv::Mat& aligned_face) +{ + if(this->aligned_face_image.empty()) + { + this->aligned_face_image = aligned_face; + } + else + { + cv::vconcat(this->aligned_face_image, aligned_face, this->aligned_face_image); + } +} + +void Visualizer::SetObservationHOG(const cv::Mat_& hog_descriptor, int num_cols, int num_rows) +{ + if(vis_hog) + { + if (this->hog_image.empty()) + { + Visualise_FHOG(hog_descriptor, num_rows, num_cols, this->hog_image); + } + else + { + cv::Mat tmp_hog; + Visualise_FHOG(hog_descriptor, num_rows, num_cols, tmp_hog); + cv::vconcat(this->hog_image, tmp_hog, this->hog_image); + } + } + +} + + +void Visualizer::SetObservationLandmarks(const cv::Mat_& landmarks_2D, double confidence, bool success, const cv::Mat_& visibilities) +{ + + if(confidence > visualisation_boundary) + { + // Draw 2D landmarks on the image + int n = landmarks_2D.rows / 2; + + // Drawing feature points + for (int i = 0; i < n; ++i) + { + if (visibilities.empty() || visibilities.at(i)) + { + cv::Point featurePoint(cvRound(landmarks_2D.at(i) * (double)draw_multiplier), cvRound(landmarks_2D.at(i + n) * (double)draw_multiplier)); + + // A rough heuristic for drawn point size + int thickness = (int)std::ceil(3.0* ((double)captured_image.cols) / 640.0); + int thickness_2 = (int)std::ceil(1.0* ((double)captured_image.cols) / 640.0); + + cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); + cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); + + } + } + } +} + +void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence) +{ + + + + // Only draw if the reliability is reasonable, the value is slightly ad-hoc + if (confidence > visualisation_boundary) + { + double vis_certainty = confidence; + if (vis_certainty > 1) + vis_certainty = 1; + + // Scale from 0 to 1, to allow to indicated by colour how confident we are in the tracking + vis_certainty = (vis_certainty - visualisation_boundary) / (1 - visualisation_boundary); + + // A rough heuristic for box around the face width + int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); + + // Draw it in reddish if uncertain, blueish if certain + DrawBox(captured_image, pose, cv::Scalar(vis_certainty*255.0, 0, (1 - vis_certainty) * 255), thickness, fx, fy, cx, cy); + } +} + +// Eye gaze infomration drawing, first of eye landmarks then of gaze +void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, double confidence) +{ + if(confidence > visualisation_boundary) + { + if (eye_landmarks2d.size() > 0) + { + // First draw the eye region landmarks + for (size_t i = 0; i < eye_landmarks2d.size(); ++i) + { + cv::Point featurePoint(cvRound(eye_landmarks2d[i].x * (double)draw_multiplier), cvRound(eye_landmarks2d[i].y * (double)draw_multiplier)); + + // A rough heuristic for drawn point size + int thickness = 1; + int thickness_2 = 1; + + size_t next_point = i + 1; + if (i == 7) + next_point = 0; + if (i == 19) + next_point = 8; + if (i == 27) + next_point = 20; + + if (i == 7 + 28) + next_point = 0 + 28; + if (i == 19 + 28) + next_point = 8 + 28; + if (i == 27 + 28) + next_point = 20 + 28; + + cv::Point nextFeaturePoint(cvRound(eye_landmarks2d[next_point].x * (double)draw_multiplier), cvRound(eye_landmarks2d[next_point].y * (double)draw_multiplier)); + if ((i < 28 && (i < 8 || i > 19)) || (i >= 28 && (i < 8 + 28 || i > 19 + 28))) + cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); + else + cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA, draw_shiftbits); + + } + + // Now draw the gaze lines themselves + cv::Mat cameraMat = (cv::Mat_(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0); + + // Grabbing the pupil location, to draw eye gaze need to know where the pupil is + cv::Point3d pupil_left(0, 0, 0); + cv::Point3d pupil_right(0, 0, 0); + for (size_t i = 0; i < 8; ++i) + { + pupil_left = pupil_left + eye_landmarks3d[i]; + pupil_right = pupil_right + eye_landmarks3d[i + eye_landmarks3d.size()/2]; + } + pupil_left = pupil_left / 8; + pupil_right = pupil_right / 8; + + std::vector points_left; + points_left.push_back(cv::Point3d(pupil_left)); + points_left.push_back(cv::Point3d(pupil_left + cv::Point3d(gaze_direction0)*50.0)); + + std::vector points_right; + points_right.push_back(cv::Point3d(pupil_right)); + points_right.push_back(cv::Point3d(pupil_right + cv::Point3d(gaze_direction1)*50.0)); + + cv::Mat_ proj_points; + cv::Mat_ mesh_0 = (cv::Mat_(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z); + Project(proj_points, mesh_0, fx, fy, cx, cy); + cv::line(captured_image, cv::Point(cvRound(proj_points.at(0, 0) * (double)draw_multiplier), cvRound(proj_points.at(0, 1) * (double)draw_multiplier)), + cv::Point(cvRound(proj_points.at(1, 0) * (double)draw_multiplier), cvRound(proj_points.at(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits); + + cv::Mat_ mesh_1 = (cv::Mat_(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z); + Project(proj_points, mesh_1, fx, fy, cx, cy); + cv::line(captured_image, cv::Point(cvRound(proj_points.at(0, 0) * (double)draw_multiplier), cvRound(proj_points.at(0, 1) * (double)draw_multiplier)), + cv::Point(cvRound(proj_points.at(1, 0) * (double)draw_multiplier), cvRound(proj_points.at(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits); + + } + } +} + +void Visualizer::SetFps(double fps) +{ + // Write out the framerate on the image before displaying it + char fpsC[255]; + std::sprintf(fpsC, "%d", (int)fps); + std::string fpsSt("FPS:"); + fpsSt += fpsC; + cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); +} + +char Visualizer::ShowObservation() +{ + if (vis_track) + { + cv::namedWindow("tracking_result", 1); + cv::imshow("tracking_result", captured_image); + } + if (vis_align) + { + cv::imshow("sim_warp", aligned_face_image); + } + if (vis_hog) + { + cv::imshow("hog", hog_image); + } + return cv::waitKey(1); +} + +cv::Mat Visualizer::GetVisImage() +{ + return captured_image; +} diff --git a/matlab_runners/Action Unit Experiments/helpers/extract_SEMAINE_labels.m b/matlab_runners/Action Unit Experiments/helpers/extract_SEMAINE_labels.m index 76ff839..074ca82 100644 --- a/matlab_runners/Action Unit Experiments/helpers/extract_SEMAINE_labels.m +++ b/matlab_runners/Action Unit Experiments/helpers/extract_SEMAINE_labels.m @@ -1,4 +1,4 @@ -function [ labels, valid_ids, vid_ids ] = extract_SEMAINE_labels( SEMAINE_dir, recs, aus ) +function [ labels, valid_ids, vid_ids, vid_names ] = extract_SEMAINE_labels( SEMAINE_dir, recs, aus ) %EXTRACT_SEMAINE_LABELS Summary of this function goes here % Detailed explanation goes here @@ -16,6 +16,7 @@ function [ labels, valid_ids, vid_ids ] = extract_SEMAINE_labels( SEMAINE_dir, labels = cell(numel(recs), 1); valid_ids = cell(numel(recs), 1); + vid_names = cell(numel(recs), 1); vid_ids = zeros(numel(recs), 2); for i=1:numel(recs) @@ -24,6 +25,9 @@ function [ labels, valid_ids, vid_ids ] = extract_SEMAINE_labels( SEMAINE_dir, vid_ids(i,:) = dlmread([SEMAINE_dir, '/', recs{i}, '.txt'], ' '); + vid_names_c = dir([SEMAINE_dir, '/', recs{i}, '/*.avi']); + [~, vid_names{i},~] = fileparts(vid_names_c.name); + xml_file = [SEMAINE_dir, recs{i}, '\' file.name]; [root_xml, name_xml, ~] = fileparts(xml_file); diff --git a/matlab_runners/Action Unit Experiments/helpers/find_BP4D.m b/matlab_runners/Action Unit Experiments/helpers/find_BP4D.m index 9266a17..036efa9 100644 --- a/matlab_runners/Action Unit Experiments/helpers/find_BP4D.m +++ b/matlab_runners/Action Unit Experiments/helpers/find_BP4D.m @@ -16,6 +16,9 @@ elseif(exist('I:\datasets\FERA_2015\BP4D\AUCoding/', 'file')) elseif(exist('D:/fera_2015/bp4d/AUCoding/', 'file')) BP4D_dir = 'D:/fera_2015/bp4d/AUCoding/'; BP4D_dir_int = 'D:/fera_2015/bp4d/AU Intensity Codes3.0/'; +elseif(exist('/media/tadas/2EBEA130BEA0F20F/datasets/FERA_2015/BP4D', 'file')) + BP4D_dir = '/media/tadas/2EBEA130BEA0F20F/datasets/FERA_2015/BP4D/AUCoding/'; + BP4D_dir_int = '/media/tadas/2EBEA130BEA0F20F/datasets/FERA_2015/BP4D/AU Intensity Codes3.0/'; else fprintf('BP4D location not found (or not defined)\n'); end diff --git a/matlab_runners/Action Unit Experiments/helpers/find_Bosphorus.m b/matlab_runners/Action Unit Experiments/helpers/find_Bosphorus.m index f392bd3..ced957d 100644 --- a/matlab_runners/Action Unit Experiments/helpers/find_Bosphorus.m +++ b/matlab_runners/Action Unit Experiments/helpers/find_Bosphorus.m @@ -2,7 +2,9 @@ if(exist('D:/Datasets/Bosphorus/', 'file')) Bosphorus_dir = 'D:\Datasets\Bosphorus/'; elseif(exist('E:/Datasets/Bosphorus/', 'file')) Bosphorus_dir = 'E:\Datasets\Bosphorus/'; -else +elseif(exist('/media/tadas/2EBEA130BEA0F20F/datasets/Bosphorus/', 'file')) + Bosphorus_dir = '/media/tadas/2EBEA130BEA0F20F/datasets/Bosphorus/'; +else fprintf('Bosphorus dataset location not found (or not defined)\n'); end diff --git a/matlab_runners/Action Unit Experiments/helpers/find_FERA2011.m b/matlab_runners/Action Unit Experiments/helpers/find_FERA2011.m index 4b760a5..39550f3 100644 --- a/matlab_runners/Action Unit Experiments/helpers/find_FERA2011.m +++ b/matlab_runners/Action Unit Experiments/helpers/find_FERA2011.m @@ -4,6 +4,8 @@ elseif(exist('/multicomp/datasets/fera2011/', 'file')) FERA2011_dir = '/multicomp/datasets/fera2011/au_training/'; elseif(exist('E:\Datasets\fera/au_training', 'file')) FERA2011_dir = 'E:\Datasets\fera/au_training/'; +elseif(exist('/media/tadas/2EBEA130BEA0F20F/datasets/fera/au_training/', 'file')) + FERA2011_dir = '/media/tadas/2EBEA130BEA0F20F/datasets/fera/au_training/'; else fprintf('FERA2011 location not found (or not defined)\n'); end diff --git a/matlab_runners/Action Unit Experiments/helpers/find_SEMAINE.m b/matlab_runners/Action Unit Experiments/helpers/find_SEMAINE.m index 6c44599..1bf6f01 100644 --- a/matlab_runners/Action Unit Experiments/helpers/find_SEMAINE.m +++ b/matlab_runners/Action Unit Experiments/helpers/find_SEMAINE.m @@ -12,6 +12,8 @@ elseif(exist('D:/fera_2015/semaine/SEMAINE-Sessions/', 'file')) SEMAINE_dir = 'D:/fera_2015/semaine/SEMAINE-Sessions/'; elseif(exist('/multicomp/datasets/face_datasets/FERA_2015/Semaine/SEMAINE-Sessions/', 'file')) SEMAINE_dir = '/multicomp/datasets/face_datasets/FERA_2015/Semaine/SEMAINE-Sessions/'; +elseif(exist('/media/tadas/2EBEA130BEA0F20F/datasets/FERA_2015/semaine/SEMAINE-Sessions/', 'file')) + SEMAINE_dir = '/media/tadas/2EBEA130BEA0F20F/datasets/FERA_2015/semaine/SEMAINE-Sessions/'; else fprintf('SEMAINE location not found (or not defined)\n'); end diff --git a/matlab_runners/Action Unit Experiments/results/BP4D_valid_res_class.txt b/matlab_runners/Action Unit Experiments/results/BP4D_valid_res_class.txt index 6f597fa..47205c9 100644 --- a/matlab_runners/Action Unit Experiments/results/BP4D_valid_res_class.txt +++ b/matlab_runners/Action Unit Experiments/results/BP4D_valid_res_class.txt @@ -1,5 +1,5 @@ AU1 class, Precision - 0.555, Recall - 0.533, F1 - 0.544 -AU2 class, Precision - 0.403, Recall - 0.440, F1 - 0.420 +AU2 class, Precision - 0.403, Recall - 0.439, F1 - 0.420 AU4 class, Precision - 0.491, Recall - 0.513, F1 - 0.502 AU6 class, Precision - 0.741, Recall - 0.776, F1 - 0.758 AU7 class, Precision - 0.764, Recall - 0.727, F1 - 0.745 diff --git a/matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt b/matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt index 2952f05..d7792c0 100644 --- a/matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt +++ b/matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt @@ -1,17 +1,17 @@ -AU1 class, Precision - 0.393, Recall - 0.727, F1 - 0.510 -AU2 class, Precision - 0.266, Recall - 0.850, F1 - 0.405 -AU4 class, Precision - 0.511, Recall - 0.874, F1 - 0.645 -AU5 class, Precision - 0.294, Recall - 0.968, F1 - 0.451 -AU6 class, Precision - 0.346, Recall - 0.833, F1 - 0.489 -AU7 class, Precision - 0.793, Recall - 0.750, F1 - 0.771 -AU9 class, Precision - 0.316, Recall - 0.960, F1 - 0.475 -AU10 class, Precision - 0.349, Recall - 0.773, F1 - 0.481 -AU12 class, Precision - 0.674, Recall - 0.864, F1 - 0.757 -AU14 class, Precision - 0.183, Recall - 0.863, F1 - 0.302 -AU15 class, Precision - 0.183, Recall - 0.851, F1 - 0.302 -AU17 class, Precision - 0.293, Recall - 0.889, F1 - 0.441 -AU20 class, Precision - 0.114, Recall - 0.930, F1 - 0.203 -AU23 class, Precision - 0.107, Recall - 0.889, F1 - 0.191 -AU25 class, Precision - 0.860, Recall - 0.873, F1 - 0.866 -AU26 class, Precision - 0.359, Recall - 0.811, F1 - 0.498 -AU45 class, Precision - 0.318, Recall - 0.771, F1 - 0.450 +AU1 class, Precision - 0.434, Recall - 0.673, F1 - 0.528 +AU2 class, Precision - 0.298, Recall - 0.818, F1 - 0.437 +AU4 class, Precision - 0.564, Recall - 0.861, F1 - 0.681 +AU5 class, Precision - 0.387, Recall - 0.915, F1 - 0.544 +AU6 class, Precision - 0.355, Recall - 0.811, F1 - 0.494 +AU7 class, Precision - 0.778, Recall - 0.783, F1 - 0.780 +AU9 class, Precision - 0.370, Recall - 0.953, F1 - 0.533 +AU10 class, Precision - 0.340, Recall - 0.788, F1 - 0.475 +AU12 class, Precision - 0.690, Recall - 0.842, F1 - 0.758 +AU14 class, Precision - 0.185, Recall - 0.881, F1 - 0.305 +AU15 class, Precision - 0.171, Recall - 0.851, F1 - 0.285 +AU17 class, Precision - 0.309, Recall - 0.861, F1 - 0.455 +AU20 class, Precision - 0.130, Recall - 0.921, F1 - 0.228 +AU23 class, Precision - 0.104, Recall - 0.837, F1 - 0.186 +AU25 class, Precision - 0.869, Recall - 0.860, F1 - 0.865 +AU26 class, Precision - 0.368, Recall - 0.809, F1 - 0.506 +AU45 class, Precision - 0.367, Recall - 0.754, F1 - 0.494 diff --git a/matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt b/matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt index 7ea7bf3..c066a02 100644 --- a/matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt +++ b/matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt @@ -1,17 +1,17 @@ -AU1 intensity, Corr - 0.717, RMS - 0.892, CCC - 0.668 -AU2 intensity, Corr - 0.696, RMS - 0.774, CCC - 0.625 -AU4 intensity, Corr - 0.802, RMS - 0.603, CCC - 0.776 -AU5 intensity, Corr - 0.747, RMS - 0.832, CCC - 0.640 -AU6 intensity, Corr - 0.556, RMS - 0.735, CCC - 0.533 -AU7 intensity, Corr - 0.831, RMS - 0.757, CCC - 0.804 -AU9 intensity, Corr - 0.779, RMS - 0.551, CCC - 0.738 -AU10 intensity, Corr - 0.495, RMS - 0.719, CCC - 0.475 -AU12 intensity, Corr - 0.810, RMS - 0.714, CCC - 0.753 -AU14 intensity, Corr - 0.348, RMS - 0.896, CCC - 0.280 -AU15 intensity, Corr - 0.527, RMS - 0.538, CCC - 0.448 -AU17 intensity, Corr - 0.561, RMS - 0.882, CCC - 0.484 -AU20 intensity, Corr - 0.413, RMS - 0.880, CCC - 0.285 -AU23 intensity, Corr - 0.354, RMS - 0.753, CCC - 0.268 -AU25 intensity, Corr - 0.847, RMS - 0.818, CCC - 0.811 -AU26 intensity, Corr - 0.514, RMS - 0.955, CCC - 0.465 -AU45 intensity, Corr - 0.868, RMS - 0.550, CCC - 0.848 +AU1 intensity, Corr - 0.712, RMS - 0.925, CCC - 0.652 +AU2 intensity, Corr - 0.696, RMS - 0.772, CCC - 0.626 +AU4 intensity, Corr - 0.797, RMS - 0.623, CCC - 0.764 +AU5 intensity, Corr - 0.767, RMS - 0.740, CCC - 0.694 +AU6 intensity, Corr - 0.541, RMS - 0.786, CCC - 0.506 +AU7 intensity, Corr - 0.830, RMS - 0.750, CCC - 0.811 +AU9 intensity, Corr - 0.763, RMS - 0.611, CCC - 0.701 +AU10 intensity, Corr - 0.491, RMS - 0.759, CCC - 0.461 +AU12 intensity, Corr - 0.804, RMS - 0.715, CCC - 0.766 +AU14 intensity, Corr - 0.357, RMS - 0.931, CCC - 0.277 +AU15 intensity, Corr - 0.516, RMS - 0.565, CCC - 0.431 +AU17 intensity, Corr - 0.554, RMS - 0.893, CCC - 0.477 +AU20 intensity, Corr - 0.411, RMS - 0.901, CCC - 0.277 +AU23 intensity, Corr - 0.351, RMS - 0.736, CCC - 0.274 +AU25 intensity, Corr - 0.846, RMS - 0.809, CCC - 0.822 +AU26 intensity, Corr - 0.516, RMS - 0.995, CCC - 0.453 +AU45 intensity, Corr - 0.840, RMS - 0.662, CCC - 0.791 diff --git a/matlab_runners/Action Unit Experiments/results/DISFA_valid_res.txt b/matlab_runners/Action Unit Experiments/results/DISFA_valid_res.txt index 90dd105..884b143 100644 --- a/matlab_runners/Action Unit Experiments/results/DISFA_valid_res.txt +++ b/matlab_runners/Action Unit Experiments/results/DISFA_valid_res.txt @@ -9,4 +9,4 @@ AU15 results - corr 0.745, rms 0.269, ccc - 0.712 AU17 results - corr 0.642, rms 0.517, ccc - 0.574 AU20 results - corr 0.619, rms 0.311, ccc - 0.581 AU25 results - corr 0.926, rms 0.500, ccc - 0.920 -AU26 results - corr 0.803, rms 0.449, ccc - 0.762 +AU26 results - corr 0.802, rms 0.449, ccc - 0.762 diff --git a/matlab_runners/Action Unit Experiments/run_AU_prediction_BP4D.m b/matlab_runners/Action Unit Experiments/run_AU_prediction_BP4D.m index 86e05d3..e2a6767 100644 --- a/matlab_runners/Action Unit Experiments/run_AU_prediction_BP4D.m +++ b/matlab_runners/Action Unit Experiments/run_AU_prediction_BP4D.m @@ -4,12 +4,12 @@ find_BP4D; BP4D_dir = [BP4D_dir, '../BP4D-training/']; out_loc = './out_bp4d/'; -if(~exist(out_loc, 'dir')) - mkdir(out_loc); -end - %% -executable = '"../../x64/Release/FeatureExtraction.exe"'; +if(isunix) + executable = '"../../build/bin/FeatureExtraction"'; +else + executable = '"../../x64/Release/FeatureExtraction.exe"'; +end bp4d_dirs = {'F002', 'F004', 'F006', 'F008', 'F010', 'F012', 'F014', 'F016', 'F018', 'F020', 'F022', 'M002', 'M004', 'M006', 'M008', 'M010', 'M012', 'M014', 'M016', 'M018'}; @@ -23,7 +23,7 @@ new_bp4d_dirs = {}; for i = 1:numel(bp4d_dirs) dirs = dir([BP4D_dir, '/', bp4d_dirs{i}, '/T*']); - tmp_dir = [BP4D_dir, '/../tmp/', bp4d_dirs{i}, '/']; + tmp_dir = [BP4D_dir, '/../tmp/', bp4d_dirs{i}]; new_bp4d_dirs = cat(1, new_bp4d_dirs, tmp_dir); if(~exist(tmp_dir, 'file')) @@ -51,18 +51,15 @@ for i = 1:numel(bp4d_dirs) end %% -parfor f1=1:numel(new_bp4d_dirs) +for f1=1:numel(new_bp4d_dirs) - command = [executable ' -asvid -no2Dfp -no3Dfp -noMparams -noPose -noGaze ']; - - [f,~,~] = fileparts(new_bp4d_dirs{f1}); - [~,f,~] = fileparts(f); - output_file = [out_loc f '.au.txt']; - - command = cat(2, command, [' -fdir "' new_bp4d_dirs{f1} '" -of "' output_file '"']); - - dos(command); + command = sprintf('%s -aus -fdir "%s" -out_dir "%s"', executable, new_bp4d_dirs{f1}, out_loc); + if(isunix) + unix(command, '-echo') + else + dos(command); + end end %% @@ -76,7 +73,7 @@ aus_BP4D = [1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23]; labels_gt = cat(1, labels_gt{:}); %% Identifying which column IDs correspond to which AU -tab = readtable([out_loc, bp4d_dirs{1}, '.au.txt']); +tab = readtable([out_loc, bp4d_dirs{1}, '.csv']); column_names = tab.Properties.VariableNames; % As there are both classes and intensities list and evaluate both of them @@ -110,10 +107,9 @@ preds_all_class = []; for i=1:numel(new_bp4d_dirs) - [f,~,~] = fileparts(new_bp4d_dirs{i}); - [~,f,~] = fileparts(f); + [~,f,~] = fileparts(new_bp4d_dirs{i}); - fname = [out_loc, f, '.au.txt']; + fname = [out_loc, f, '.csv']; preds = dlmread(fname, ',', 1, 0); @@ -157,7 +153,7 @@ valid_ids = cat(1, valid_ids{:}); labels_gt = cat(1, labels_gt{:}); %% Identifying which column IDs correspond to which AU -tab = readtable([out_loc, bp4d_dirs{1}, '.au.txt']); +tab = readtable([out_loc, bp4d_dirs{1}, '.csv']); column_names = tab.Properties.VariableNames; % As there are both classes and intensities list and evaluate both of them @@ -184,10 +180,9 @@ preds_all_int = []; for i=1:numel(new_bp4d_dirs) - [f,~,~] = fileparts(new_bp4d_dirs{i}); - [~,f,~] = fileparts(f); - - fname = [out_loc, f, '.au.txt']; + [~,f,~] = fileparts(new_bp4d_dirs{i}); + + fname = [out_loc, f, '.csv']; preds = dlmread(fname, ',', 1, 0); % Read all of the intensity AUs diff --git a/matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m b/matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m index 3c5e745..b60e3bf 100644 --- a/matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m +++ b/matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m @@ -7,12 +7,12 @@ addpath('./helpers'); find_Bosphorus; out_loc = './out_bosph/'; -if(~exist(out_loc, 'dir')) - mkdir(out_loc); -end - %% -executable = '"../../x64/Release/FaceLandmarkImg.exe"'; +if(isunix) + executable = '"../../build/bin/FaceLandmarkImg"'; +else + executable = '"../../x64/Release/FaceLandmarkImg.exe"'; +end bosph_dirs = dir([Bosphorus_dir, '/BosphorusDB/BosphorusDB/bs*']); @@ -22,11 +22,14 @@ parfor f1=1:numel(bosph_dirs) command = executable; input_dir = [Bosphorus_dir, '/BosphorusDB/BosphorusDB/', bosph_dirs(f1).name]; - command = cat(2, command, [' -fdir "' input_dir '" -ofdir "' out_loc '"']); - command = cat(2, command, ' -multi_view 1 -wild -q'); - - dos(command); + command = cat(2, command, [' -fdir "' input_dir '" -out_dir "' out_loc '"']); + command = cat(2, command, ' -multi_view 1 -wild -aus '); + if(isunix) + unix(command, '-echo') + else + dos(command); + end end %% @@ -37,64 +40,32 @@ aus_Bosph = [1, 2, 4, 5, 6, 7, 9, 10, 12, 14, 15, 17, 20, 23, 25, 26, 45]; %% Read the predicted values -% First read the first file to get the ids and line numbers -% au occurences -fid = fopen([out_loc, filenames{1}, '_det_0.pts']); -data = fgetl(fid); - -ind = 0; -beg_ind = -1; -end_ind = -1; -aus_det = []; -aus_det_id = []; - -%% -while ischar(data) - if(~isempty(findstr(data, 'au occurences:'))) - num_occurences = str2num(data(numel('au occurences:')+1:end)); - % Skip ahead two lines - data = fgetl(fid); - data = fgetl(fid); - ind = ind + 2; - beg_ind = ind; - end - - if(beg_ind ~= -1 && end_ind == -1) - if(~isempty(findstr(data, '}'))) - end_ind = ind; - else - d = strsplit(data, ' '); - aus_det = cat(1, aus_det, str2num(d{1}(3:end))); - aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1); - end - end - - data = fgetl(fid); - ind = ind + 1; +% First read the first file to get the column ids +tab = readtable([out_loc, filenames{1}, '.csv']); +column_names = tab.Properties.VariableNames; +aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_c')); +aus_det_cell = column_names(aus_det_id); +aus_det = zeros(size(aus_det_cell)); +for i=1:numel(aus_det) + aus_det(i) = str2num(aus_det_cell{i}(3:4)); end -fclose(fid); %% labels_pred = zeros(size(labels_gt)); for i=1:numel(filenames) % Will need to read the relevant AUs only - if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file')) - fid = fopen([out_loc, filenames{i}, '_det_0.pts']); - for k=1:beg_ind - data = fgetl(fid); + all_params = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0); + + % if multiple faces detected just take the first row + aus_pred = all_params(1, aus_det_id); + + for k=1:numel(aus_det) + if(sum(aus_Bosph == aus_det(k))>0) + labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k); end - - for k=1:num_occurences - data = fgetl(fid); - if(sum(aus_Bosph == aus_det(k))>0) - d = strsplit(data, ' '); - labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2}); - end - end - - fclose(fid); end + end %% @@ -122,63 +93,32 @@ fclose(f); %% Read the predicted values for intensities -% First read the first file to get the ids and line numbers -% au occurences -fid = fopen([out_loc, filenames{1}, '_det_0.pts']); -data = fgetl(fid); - -ind = 0; -beg_ind = -1; -end_ind = -1; -aus_det = []; -aus_det_id = []; - -while ischar(data) - if(~isempty(findstr(data, 'au intensities:'))) - num_occurences = str2num(data(numel('au intensities:')+1:end)); - % Skip ahead two lines - data = fgetl(fid); - data = fgetl(fid); - ind = ind + 2; - beg_ind = ind; - end - - if(beg_ind ~= -1 && end_ind == -1) - if(~isempty(findstr(data, '}'))) - end_ind = ind; - else - d = strsplit(data, ' '); - aus_det = cat(1, aus_det, str2num(d{1}(3:end))); - aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1); - end - end - - data = fgetl(fid); - ind = ind + 1; +% First read the first file to get the column ids +tab = readtable([out_loc, filenames{1}, '.csv']); +column_names = tab.Properties.VariableNames; +aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r')); +aus_det_cell = column_names(aus_det_id); +aus_det = zeros(size(aus_det_cell)); +for i=1:numel(aus_det) + aus_det(i) = str2num(aus_det_cell{i}(3:4)); end -fclose(fid); %% labels_pred = zeros(size(labels_gt)); for i=1:numel(filenames) % Will need to read the relevant AUs only - if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file')) - fid = fopen([out_loc, filenames{i}, '_det_0.pts']); - for k=1:beg_ind - data = fgetl(fid); + all_params = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0); + + % if multiple faces detected just take the first row + aus_pred = all_params(1, aus_det_id); + + for k=1:numel(aus_det) + if(sum(aus_Bosph == aus_det(k))>0) + labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k); end - - for k=1:num_occurences - data = fgetl(fid); - if(sum(aus_Bosph == aus_det(k))>0) - d = strsplit(data, ' '); - labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2}); - end - end - - fclose(fid); end + end %% diff --git a/matlab_runners/Action Unit Experiments/run_AU_prediction_DISFA.m b/matlab_runners/Action Unit Experiments/run_AU_prediction_DISFA.m index 9eeeab5..8451caa 100644 --- a/matlab_runners/Action Unit Experiments/run_AU_prediction_DISFA.m +++ b/matlab_runners/Action Unit Experiments/run_AU_prediction_DISFA.m @@ -10,17 +10,18 @@ if(exist('D:/Datasets/DISFA/Videos_LeftCamera/', 'file')) DISFA_dir = 'D:/Datasets/DISFA/Videos_LeftCamera/'; elseif(exist('E:/Datasets/DISFA/Videos_LeftCamera/', 'file')) DISFA_dir = 'E:/Datasets/DISFA/Videos_LeftCamera/'; -else +elseif(exist('/multicomp/datasets/face_datasets/DISFA/Videos_LeftCamera/', 'file')) DISFA_dir = '/multicomp/datasets/face_datasets/DISFA/Videos_LeftCamera/'; +elseif(exist('/media/tadas/2EBEA130BEA0F20F/datasets/DISFA/', 'file')) + DISFA_dir = '/media/tadas/2EBEA130BEA0F20F/datasets/DISFA/Videos_LeftCamera/'; +else + fprintf('Cannot find DIFA location\n'); end videos = dir([DISFA_dir, '*.avi']); output = 'out_DISFA/'; -if(~exist(output, 'file')) - mkdir(output); -end %% % Do it in parrallel for speed (replace the parfor with for if no parallel @@ -29,11 +30,7 @@ parfor v = 1:numel(videos) vid_file = [DISFA_dir, videos(v).name]; - [~, name, ~] = fileparts(vid_file); - - % where to output tracking results - output_file = [output name '_au.txt']; - command = [executable ' -f "' vid_file '" -of "' output_file '" -q -no2Dfp -no3Dfp -noMparams -noPose -noGaze']; + command = sprintf('%s -f "%s" -out_dir "%s" -aus ', executable, vid_file, output); if(isunix) unix(command, '-echo'); @@ -70,30 +67,32 @@ for i=1:numel(label_folders) label_ids = cat(1, label_ids, repmat(user_id, size(labels,1),1)); end -preds_files = dir([prediction_dir, '*SN*.txt']); +preds_files = dir([prediction_dir, '*SN*.csv']); tab = readtable([prediction_dir, preds_files(1).name]); column_names = tab.Properties.VariableNames; aus_pred_int = []; -for c=3:numel(column_names) +au_inds_in_file = []; +for c=1:numel(column_names) if(strfind(column_names{c}, '_r') > 0) aus_pred_int = cat(1, aus_pred_int, int32(str2num(column_names{c}(3:end-2)))); + au_inds_in_file = cat(1, au_inds_in_file, c); end end inds_au = zeros(numel(AUs_disfa),1); for ind=1:numel(AUs_disfa) - inds_au(ind) = find(aus_pred_int==AUs_disfa(ind)); + inds_au(ind) = au_inds_in_file(aus_pred_int==AUs_disfa(ind)); end preds_all = zeros(size(labels_all,1), numel(AUs_disfa)); for i=1:numel(preds_files) preds = dlmread([prediction_dir, preds_files(i).name], ',', 1, 0); - preds = preds(:,5:5+numel(aus_pred_int)-1); + %preds = preds(:,5:5+numel(aus_pred_int)-1); - user_id = str2num(preds_files(i).name(end - 14:end-12)); + user_id = str2num(preds_files(i).name(end - 11:end-9)); rel_ids = label_ids == user_id; preds_all(rel_ids,:) = preds(:,inds_au); end diff --git a/matlab_runners/Action Unit Experiments/run_AU_prediction_FERA2011.m b/matlab_runners/Action Unit Experiments/run_AU_prediction_FERA2011.m index 4daf563..5db47d4 100644 --- a/matlab_runners/Action Unit Experiments/run_AU_prediction_FERA2011.m +++ b/matlab_runners/Action Unit Experiments/run_AU_prediction_FERA2011.m @@ -5,10 +5,6 @@ find_FERA2011; out_loc = './out_fera/'; -if(~exist(out_loc, 'dir')) - mkdir(out_loc); -end - %% if(isunix) executable = '"../../build/bin/FeatureExtraction"'; @@ -18,20 +14,17 @@ end fera_dirs = dir([FERA2011_dir, 'train*']); -parfor f1=1:numel(fera_dirs) +for f1=1:numel(fera_dirs) vid_files = dir([FERA2011_dir, fera_dirs(f1).name, '/*.avi']); for v=1:numel(vid_files) - command = [executable ' -asvid -q -no2Dfp -no3Dfp -noMparams -noPose -noGaze -au_static ']; + command = [executable ' -aus -au_static ']; curr_vid = [FERA2011_dir, fera_dirs(f1).name, '/', vid_files(v).name]; - [~,name,~] = fileparts(curr_vid); - output_file = [out_loc name '.au.txt']; - - command = cat(2, command, [' -f "' curr_vid '" -of "' output_file '"']); + command = cat(2, command, [' -f "' curr_vid '" -out_dir "' out_loc '"']); if(isunix) unix(command, '-echo'); @@ -50,7 +43,7 @@ for i=1:numel(filenames) end %% Identifying which column IDs correspond to which AU -tab = readtable([out_loc, 'train_001.au.txt']); +tab = readtable([out_loc, 'train_001.csv']); column_names = tab.Properties.VariableNames; % As there are both classes and intensities list and evaluate both of them @@ -79,7 +72,7 @@ preds_all_class = []; for i=1:numel(filenames) - fname = dir([out_loc, '/*', filenames{i}, '.au.txt']); + fname = dir([out_loc, '/*', filenames{i}, '.csv']); fname = fname(1).name; preds = dlmread([out_loc '/' fname], ',', 1, 0); diff --git a/matlab_runners/Action Unit Experiments/run_AU_prediction_SEMAINE.m b/matlab_runners/Action Unit Experiments/run_AU_prediction_SEMAINE.m index ef862fe..6bcefc7 100644 --- a/matlab_runners/Action Unit Experiments/run_AU_prediction_SEMAINE.m +++ b/matlab_runners/Action Unit Experiments/run_AU_prediction_SEMAINE.m @@ -5,10 +5,6 @@ find_SEMAINE; out_loc = './out_SEMAINE/'; -if(~exist(out_loc, 'dir')) - mkdir(out_loc); -end - if(isunix) executable = '"../../build/bin/FeatureExtraction"'; else @@ -24,14 +20,9 @@ parfor f1=1:numel(devel_recs) f1_dir = devel_recs{f1}; - command = [executable, ' -fx 800 -fy 800 -q -no2Dfp -no3Dfp -noMparams -noPose -noGaze ']; - curr_vid = [SEMAINE_dir, f1_dir, '/', vid_file.name]; - name = f1_dir; - output_aus = [out_loc name '.au.txt']; - - command = cat(2, command, [' -f "' curr_vid '" -of "' output_aus, '"']); + command = sprintf('%s -aus -f "%s" -out_dir "%s" ', executable, curr_vid, out_loc); if(isunix) unix(command, '-echo'); @@ -43,26 +34,19 @@ parfor f1=1:numel(devel_recs) end %% Actual model evaluation -[ labels, valid_ids, vid_ids ] = extract_SEMAINE_labels(SEMAINE_dir, devel_recs, aus_SEMAINE); +[ labels, valid_ids, vid_ids, vid_names ] = extract_SEMAINE_labels(SEMAINE_dir, devel_recs, aus_SEMAINE); labels_gt = cat(1, labels{:}); %% Identifying which column IDs correspond to which AU -tab = readtable([out_loc, devel_recs{1}, '.au.txt']); +tab = readtable([out_loc, vid_names{1}, '.csv']); column_names = tab.Properties.VariableNames; % As there are both classes and intensities list and evaluate both of them -aus_pred_int = []; aus_pred_class = []; - -inds_int_in_file = []; inds_class_in_file = []; for c=1:numel(column_names) - if(strfind(column_names{c}, '_r') > 0) - aus_pred_int = cat(1, aus_pred_int, int32(str2num(column_names{c}(3:end-2)))); - inds_int_in_file = cat(1, inds_int_in_file, c); - end if(strfind(column_names{c}, '_c') > 0) aus_pred_class = cat(1, aus_pred_class, int32(str2num(column_names{c}(3:end-2)))); inds_class_in_file = cat(1, inds_class_in_file, c); @@ -70,37 +54,20 @@ for c=1:numel(column_names) end %% -inds_au_int = zeros(size(aus_SEMAINE)); inds_au_class = zeros(size(aus_SEMAINE)); -for ind=1:numel(aus_SEMAINE) - if(~isempty(find(aus_pred_int==aus_SEMAINE(ind), 1))) - inds_au_int(ind) = find(aus_pred_int==aus_SEMAINE(ind)); - end -end - for ind=1:numel(aus_SEMAINE) if(~isempty(find(aus_pred_class==aus_SEMAINE(ind), 1))) - inds_au_class(ind) = find(aus_pred_class==aus_SEMAINE(ind)); + inds_au_class(ind) = inds_class_in_file(aus_pred_class==aus_SEMAINE(ind)); end end -preds_all_class = []; -preds_all_int = []; - -for i=1:numel(devel_recs) +preds_all = []; +for i=1:numel(vid_names) - fname = [out_loc, devel_recs{i}, '.au.txt']; + fname = [out_loc, vid_names{i}, '.csv']; preds = dlmread(fname, ',', 1, 0); - - % Read all of the intensity AUs - preds_int = preds(vid_ids(i,1):vid_ids(i,2) - 1, inds_int_in_file); - - % Read all of the classification AUs - preds_class = preds(vid_ids(i,1):vid_ids(i,2) - 1, inds_class_in_file); - - preds_all_class = cat(1, preds_all_class, preds_class); - preds_all_int = cat(1, preds_all_int, preds_int); + preds_all = cat(1, preds_all, preds(vid_ids(i,1):vid_ids(i,2) - 1, :)); end %% @@ -109,10 +76,10 @@ f1s = zeros(1, numel(aus_SEMAINE)); for au = 1:numel(aus_SEMAINE) if(inds_au_class(au) ~= 0) - tp = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 1); - fp = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 1); - fn = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 0); - tn = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 0); + tp = sum(labels_gt(:,au) == 1 & preds_all(:, inds_au_class(au)) == 1); + fp = sum(labels_gt(:,au) == 0 & preds_all(:, inds_au_class(au)) == 1); + fn = sum(labels_gt(:,au) == 1 & preds_all(:, inds_au_class(au)) == 0); + tn = sum(labels_gt(:,au) == 0 & preds_all(:, inds_au_class(au)) == 0); precision = tp./(tp+fp); recall = tp./(tp+fn); diff --git a/matlab_runners/Demos/feature_extraction_demo_img_seq.m b/matlab_runners/Demos/feature_extraction_demo_img_seq.m index 5ac718c..13120e6 100644 --- a/matlab_runners/Demos/feature_extraction_demo_img_seq.m +++ b/matlab_runners/Demos/feature_extraction_demo_img_seq.m @@ -1,44 +1,25 @@ +% A demo script that demonstrates how to process a single video file using +% OpenFace and extract and visualize all of the features + clear +% The location executable will depend on the OS if(isunix) executable = '"../../build/bin/FeatureExtraction"'; else executable = '"../../x64/Release/FeatureExtraction.exe"'; end -output = './output_features_seq/'; +% Input file +in_dir = '../../samples/image_sequence'; -if(~exist(output, 'file')) - mkdir(output) -end - -in_dirs = {'../../image_sequence'}; -% some parameters -verbose = true; +% Where to store the output +output_dir = './processed_features/'; -command = executable; - -% Remove for a speedup -command = cat(2, command, ' -verbose '); - -% add all videos to single argument list (so as not to load the model anew -% for every video) -for i=1:numel(in_dirs) - - [~, name, ~] = fileparts(in_dirs{i}); - - % where to output tracking results - outputFile = [output name '.txt']; - outputDir_aligned = [output name]; - - outputHOG_aligned = [output name '.hog']; - - command = cat(2, command, ['-asvid -fdir "' in_dirs{i} '" -of "' outputFile '" ']); - - command = cat(2, command, [' -simalign "' outputDir_aligned '" -simsize 200 -hogalign "' outputHOG_aligned '"']); +% This will take directory after -fdir and output all the features to directory +% after -out_dir +command = sprintf('%s -fdir "%s" -out_dir "%s" -verbose', executable, in_dir, output_dir); -end - if(isunix) unix(command); else @@ -47,15 +28,30 @@ end %% Demonstrating reading the output files -% First read in the column names -tab = readtable(outputFile); +% Most of the features will be in the csv file in the output directory with +% the same name as the input file +[~,name,~] = fileparts(in_dir); +output_csv = sprintf('%s/%s.csv', output_dir, name); + +% First read in the column names, to know which columns to read for +% particular features +tab = readtable(output_csv); column_names = tab.Properties.VariableNames; -all_params = dlmread(outputFile, ',', 1, 0); +% Read all of the data +all_params = dlmread(output_csv, ',', 1, 0); % This indicates which frames were succesfully tracked -valid_frames = logical(all_params(:,4)); -time = all_params(valid_frames, 2); + +% Find which column contains success of tracking data and timestamp data +valid_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'success')); +frame_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'frame')); + +% Extract tracking success data and only read those frame +valid_frames = logical(all_params(:,valid_ind)); + +% Get the timestamp data +frame_nums = all_params(valid_frames, frame_ind); %% Finding which header line starts with p_ (basically model params) shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_')); @@ -64,9 +60,9 @@ shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_')); shape_params = all_params(valid_frames, shape_inds); figure -plot(time, shape_params); +plot(frame_nums, shape_params); title('Shape parameters'); -xlabel('Time (s)'); +xlabel('Frame'); %% Demonstrate 2D landmarks landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'x_')); @@ -75,10 +71,20 @@ landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'y_')) xs = all_params(valid_frames, landmark_inds_x); ys = all_params(valid_frames, landmark_inds_y); +eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_x_')); +eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_y_')); + +eye_xs = all_params(valid_frames, eye_landmark_inds_x); +eye_ys = all_params(valid_frames, eye_landmark_inds_y); + figure for j = 1:size(xs,1) plot(xs(j,:), -ys(j,:), '.'); + hold on; + plot(eye_xs(j,:), -eye_ys(j,:), '.r'); + hold off; + xlim([min(xs(1,:)) * 0.5, max(xs(2,:))*1.4]); ylim([min(-ys(1,:)) * 1.4, max(-ys(2,:))*0.5]); xlabel('x (px)'); @@ -96,9 +102,20 @@ xs = all_params(valid_frames, landmark_inds_x); ys = all_params(valid_frames, landmark_inds_y); zs = all_params(valid_frames, landmark_inds_z); +eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_X_')); +eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_Y_')); +eye_landmark_inds_z = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_Z_')); + +eye_xs = all_params(valid_frames, eye_landmark_inds_x); +eye_ys = all_params(valid_frames, eye_landmark_inds_y); +eye_zs = all_params(valid_frames, eye_landmark_inds_z); + figure for j = 1:size(xs,1) plot3(xs(j,:), ys(j,:), zs(j,:), '.');axis equal; + hold on; + plot3(eye_xs(j,:), eye_ys(j,:), eye_zs(j,:), '.r'); + hold off; xlabel('X (mm)'); ylabel('Y (mm)'); zlabel('Z (mm)'); @@ -110,7 +127,7 @@ au_reg_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r')); aus = all_params(valid_frames, au_reg_inds); figure -plot(aus); +plot(frame_nums, aus); title('Facial Action Units (intensity)'); xlabel('Time (s)'); ylabel('Intensity'); @@ -120,7 +137,7 @@ au_class_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_c')); aus = all_params(valid_frames, au_class_inds); figure -plot(aus); +plot(frame_nums, aus); title('Facial Action Units (presense)'); xlabel('Time (s)'); ylim([0,2]); @@ -129,18 +146,34 @@ pose_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'pose_')); pose = all_params(valid_frames, pose_inds); figure -plot(pose); +plot(frame_nums, pose); title('Pose (rotation and translation)'); -xlabel('Time (s)'); +xlabel('Frame number'); + +%% Demo gaze +gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle')); + +% Read gaze (x,y,z) for one eye and (x,y,z) for another +gaze = all_params(valid_frames, gaze_inds); + +plot(frame_nums, gaze(:,1), 'DisplayName', 'Left - right'); +hold on; +plot(frame_nums, gaze(:,2), 'DisplayName', 'Up - down'); +xlabel('Frame number') % x-axis label +ylabel('Angle radians') % y-axis label +legend('show'); +hold off; %% Output HOG files -[hog_data, valid_inds] = Read_HOG_file(outputHOG_aligned); +output_hog_file = sprintf('%s/%s.hog', output_dir, name); +[hog_data, valid_inds] = Read_HOG_file(output_hog_file); %% Output aligned images -img_files = dir([outputDir_aligned, '/*.png']); +output_aligned_dir = sprintf('%s/%s_aligned/', output_dir, name); +img_files = dir([output_aligned_dir, '/*.bmp']); imgs = cell(numel(img_files, 1)); for i=1:numel(img_files) - imgs{i} = imread([ outputDir_aligned, '/', img_files(i).name]); + imgs{i} = imread([ output_aligned_dir, '/', img_files(i).name]); imshow(imgs{i}) drawnow end \ No newline at end of file diff --git a/matlab_runners/Demos/feature_extraction_demo_vid.m b/matlab_runners/Demos/feature_extraction_demo_vid.m index ce65c8a..810090c 100644 --- a/matlab_runners/Demos/feature_extraction_demo_vid.m +++ b/matlab_runners/Demos/feature_extraction_demo_vid.m @@ -1,49 +1,25 @@ +% A demo script that demonstrates how to process a single video file using +% OpenFace and extract and visualize all of the features + clear +% The location executable will depend on the OS if(isunix) executable = '"../../build/bin/FeatureExtraction"'; else executable = '"../../x64/Release/FeatureExtraction.exe"'; end -output = './output_features_vid/'; +% Input file +in_file = '../../samples/default.wmv'; -if(~exist(output, 'file')) - mkdir(output) -end - -in_files = dir('../../samples/default.wmv'); -% some parameters -verbose = true; +% Where to store the output +output_dir = './processed_features/'; -command = executable; - -% Remove for a speedup -command = cat(2, command, ' -verbose '); - -% add all videos to single argument list (so as not to load the model anew -% for every video) -for i=1:numel(in_files) - - inputFile = ['../../samples/', in_files(i).name]; - [~, name, ~] = fileparts(inputFile); - - % where to output tracking results - outputFile = [output name '.csv']; - - if(~exist([output name], 'file')) - mkdir([output name]); - end - - outputDir_aligned = [output name]; - - outputHOG_aligned = [output name '.hog']; - - command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '"']); - command = cat(2, command, [' -au_static -simsize 224 -simalign "' outputDir_aligned '" -hogalign "' outputHOG_aligned '"' ]); +% This will take file after -f and output all the features to directory +% after -out_dir +command = sprintf('%s -f "%s" -out_dir "%s" -verbose', executable, in_file, output_dir); -end - if(isunix) unix(command); else @@ -52,15 +28,30 @@ end %% Demonstrating reading the output files -% First read in the column names -tab = readtable(outputFile); +% Most of the features will be in the csv file in the output directory with +% the same name as the input file +[~,name,~] = fileparts(in_file); +output_csv = sprintf('%s/%s.csv', output_dir, name); + +% First read in the column names, to know which columns to read for +% particular features +tab = readtable(output_csv); column_names = tab.Properties.VariableNames; -all_params = dlmread(outputFile, ',', 1, 0); +% Read all of the data +all_params = dlmread(output_csv, ',', 1, 0); % This indicates which frames were succesfully tracked -valid_frames = logical(all_params(:,4)); -time = all_params(valid_frames, 2); + +% Find which column contains success of tracking data and timestamp data +valid_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'success')); +time_stamp_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'timestamp')); + +% Extract tracking success data and only read those frame +valid_frames = logical(all_params(:,valid_ind)); + +% Get the timestamp data +time_stamps = all_params(valid_frames, time_stamp_ind); %% Finding which header line starts with p_ (basically model params) shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_')); @@ -69,7 +60,7 @@ shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_')); shape_params = all_params(valid_frames, shape_inds); figure -plot(time, shape_params); +plot(time_stamps, shape_params); title('Shape parameters'); xlabel('Time (s)'); @@ -77,23 +68,23 @@ xlabel('Time (s)'); landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'x_')); landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'y_')); -landmark_inds_x_eye = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_x_')); -landmark_inds_y_eye = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_y_')); - xs = all_params(valid_frames, landmark_inds_x); ys = all_params(valid_frames, landmark_inds_y); -xs_eye = all_params(valid_frames, landmark_inds_x_eye); -ys_eye = all_params(valid_frames, landmark_inds_y_eye); +eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_x_')); +eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_y_')); + +eye_xs = all_params(valid_frames, eye_landmark_inds_x); +eye_ys = all_params(valid_frames, eye_landmark_inds_y); figure for j = 1:size(xs,1) plot(xs(j,:), -ys(j,:), '.'); hold on; - plot(xs_eye(j,:), -ys_eye(j,:), '.'); + plot(eye_xs(j,:), -eye_ys(j,:), '.r'); hold off; - + xlim([min(xs(1,:)) * 0.5, max(xs(2,:))*1.4]); ylim([min(-ys(1,:)) * 1.4, max(-ys(2,:))*0.5]); xlabel('x (px)'); @@ -111,9 +102,20 @@ xs = all_params(valid_frames, landmark_inds_x); ys = all_params(valid_frames, landmark_inds_y); zs = all_params(valid_frames, landmark_inds_z); +eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_X_')); +eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_Y_')); +eye_landmark_inds_z = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_Z_')); + +eye_xs = all_params(valid_frames, eye_landmark_inds_x); +eye_ys = all_params(valid_frames, eye_landmark_inds_y); +eye_zs = all_params(valid_frames, eye_landmark_inds_z); + figure for j = 1:size(xs,1) plot3(xs(j,:), ys(j,:), zs(j,:), '.');axis equal; + hold on; + plot3(eye_xs(j,:), eye_ys(j,:), eye_zs(j,:), '.r'); + hold off; xlabel('X (mm)'); ylabel('Y (mm)'); zlabel('Z (mm)'); @@ -125,7 +127,7 @@ au_reg_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r')); aus = all_params(valid_frames, au_reg_inds); figure -plot(time, aus); +plot(time_stamps, aus); title('Facial Action Units (intensity)'); xlabel('Time (s)'); ylabel('Intensity'); @@ -135,7 +137,7 @@ au_class_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_c')); aus = all_params(valid_frames, au_class_inds); figure -plot(time, aus); +plot(time_stamps, aus); title('Facial Action Units (presense)'); xlabel('Time (s)'); ylim([0,2]); @@ -144,7 +146,7 @@ pose_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'pose_')); pose = all_params(valid_frames, pose_inds); figure -plot(pose); +plot(time_stamps, pose); title('Pose (rotation and translation)'); xlabel('Time (s)'); @@ -154,22 +156,24 @@ gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle' % Read gaze (x,y,z) for one eye and (x,y,z) for another gaze = all_params(valid_frames, gaze_inds); -plot(time, gaze(:,1), 'DisplayName', 'Left - right'); +plot(time_stamps, gaze(:,1), 'DisplayName', 'Left - right'); hold on; -plot(time, gaze(:,2), 'DisplayName', 'Up - down'); +plot(time_stamps, gaze(:,2), 'DisplayName', 'Up - down'); xlabel('Time(s)') % x-axis label ylabel('Angle radians') % y-axis label legend('show'); hold off; %% Output HOG files -[hog_data, valid_inds] = Read_HOG_file(outputHOG_aligned); +output_hog_file = sprintf('%s/%s.hog', output_dir, name); +[hog_data, valid_inds] = Read_HOG_file(output_hog_file); %% Output aligned images -img_files = dir([outputDir_aligned, '/*.bmp']); +output_aligned_dir = sprintf('%s/%s_aligned/', output_dir, name); +img_files = dir([output_aligned_dir, '/*.bmp']); imgs = cell(numel(img_files, 1)); for i=1:numel(img_files) - imgs{i} = imread([ outputDir_aligned, '/', img_files(i).name]); + imgs{i} = imread([ output_aligned_dir, '/', img_files(i).name]); imshow(imgs{i}) drawnow end \ No newline at end of file diff --git a/matlab_runners/Demos/gaze_extraction_demo_vid.m b/matlab_runners/Demos/gaze_extraction_demo_vid.m index 38484d5..574f299 100644 --- a/matlab_runners/Demos/gaze_extraction_demo_vid.m +++ b/matlab_runners/Demos/gaze_extraction_demo_vid.m @@ -6,37 +6,12 @@ else executable = '"../../x64/Release/FeatureExtraction.exe"'; end -output = './output_features_vid/'; +output = './processed_features/'; + +in_file = '../../samples/2015-10-15-15-14.avi'; -if(~exist(output, 'file')) - mkdir(output) -end +command = sprintf('%s -f "%s" -out_dir "%s" -gaze -verbose', executable, in_file, output); -in_files = dir('../../samples/2015-10-15-15-14.avi'); -% some parameters -verbose = true; - -command = executable; -command = cat(2, command, ' -verbose -no2Dfp -no3Dfp -noMparams -noPose -noAUs '); - -% add all videos to single argument list (so as not to load the model anew -% for every video) -for i=1:numel(in_files) - - inputFile = ['../../samples/', in_files(i).name]; - [~, name, ~] = fileparts(inputFile); - - % where to output tracking results - outputFile_gaze = [output name '_gaze.txt']; - - if(~exist([output name], 'file')) - mkdir([output name]); - end - - command = cat(2, command, ['-fx 700 -fy 700 -f "' inputFile '" -of "' outputFile_gaze '"']); - -end - if(isunix) unix(command); else @@ -44,41 +19,23 @@ else end %% Demonstrating reading the output files -filename = [output name]; +[~, out_filename,~] = fileparts(in_file); +out_filename = sprintf("%s/%s.csv", output, out_filename); -% Read gaze (x,y,z) for one eye and (x,y,z) for another -gaze = dlmread([filename, '_gaze.txt'], ',', 1, 0); +% First read in the column names +tab = readtable(out_filename); +column_names = tab.Properties.VariableNames; -% This indicates which frames were succesfully tracked -valid_frames = gaze(:,4); +all_params = dlmread(out_filename, ',', 1, 0); -% only picking left, right and up down views for visualisation +gaze_angle_ids = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle_')); -% These are gaze angles with respect to head pose -gaze_angle = gaze(:,[11,12]); +gaze = all_params(:,gaze_angle_ids); -figure; -plot(smooth(gaze_angle(:,1)), 'DisplayName', 'Left - right'); -hold on; -plot(smooth(gaze_angle(:,2)), 'DisplayName', 'Up - down'); -xlabel('Frame') % x-axis label -ylabel('Radians') % y-axis label -legend('show'); -hold off; - -% These are gaze direction vectors -gaze = gaze(:,[5,6,7,8,9,10]); - -gaze = (gaze(:,[1,2,3]) + gaze(:,[4,5,6]))/2; -gaze(:,1) = smooth(gaze(:,1)); -gaze(:,2) = smooth(gaze(:,2)); -gaze(:,3) = smooth(gaze(:,3)); - -figure; plot(gaze(:,1), 'DisplayName', 'Left - right'); hold on; plot(gaze(:,2), 'DisplayName', 'Up - down'); xlabel('Frame') % x-axis label -ylabel('Gaze vector size') % y-axis label +ylabel('Angle in radians') % y-axis label legend('show'); hold off; \ No newline at end of file diff --git a/matlab_runners/Demos/run_demo_align_size.m b/matlab_runners/Demos/run_demo_align_size.m new file mode 100644 index 0000000..2e80267 --- /dev/null +++ b/matlab_runners/Demos/run_demo_align_size.m @@ -0,0 +1,47 @@ +% A demo script that demonstrates how to extract aligned faces from a sequence, +% also shows how to extract different sized aligned faces + +clear + +% The location executable will depend on the OS +if(isunix) + executable = '"../../build/bin/FeatureExtraction"'; +else + executable = '"../../x64/Release/FeatureExtraction.exe"'; +end + +% Input file +in_file = '../../samples/default.wmv'; + +% Where to store the output +output_dir = './processed_features/'; + +img_sizes = [64, 112, 224]; + +for s=1:numel(img_sizes) + % This will take file after -f and output all the features to directory + % after -out_dir, with name after -of + command = sprintf('%s -f "%s" -out_dir "%s" -simalign -simsize %d -of sample_%d', executable, in_file, output_dir, img_sizes(s), img_sizes(s) ); + + if(isunix) + unix(command); + else + dos(command); + end + + %% Output aligned images + output_aligned_dir = sprintf('%s/sample_%d_aligned/', output_dir, img_sizes(s)); + img_files = dir([output_aligned_dir, '/*.bmp']); + imgs = cell(numel(img_files), 1); + + assert(numel(imgs) > 0); + + for i=1:numel(img_files) + imgs{i} = imread([ output_aligned_dir, '/', img_files(i).name]); + + assert(size(imgs{i},1) == img_sizes(s) && size(imgs{i},2) == img_sizes(s)); + + imshow(imgs{i}) + drawnow + end +end \ No newline at end of file diff --git a/matlab_runners/Demos/run_demo_images.m b/matlab_runners/Demos/run_demo_images.m index 8d8c408..0af9f77 100644 --- a/matlab_runners/Demos/run_demo_images.m +++ b/matlab_runners/Demos/run_demo_images.m @@ -8,38 +8,22 @@ end in_dir = '../../samples/'; out_dir = './demo_img/'; -if(~exist(out_dir, 'file')) - mkdir(out_dir); -end +model = 'model/main_clnf_general.txt'; % Trained on in the wild and multi-pie data (a CLNF model) -% some parameters -verbose = true; +% Uncomment the below models if you want to try them +%model = 'model/main_clnf_wild.txt'; % Trained on in-the-wild data only -% Trained on in the wild and multi-pie data (less accurate CLM model) -% model = 'model/main_clm_general.txt'; -% Trained on in-the-wild -%model = 'model/main_clm_wild.txt'; +%model = 'model/main_clm_general.txt'; % Trained on in the wild and multi-pie data (less accurate SVR/CLM model) +%model = 'model/main_clm_wild.txt'; % Trained on in-the-wild -% Trained on in the wild and multi-pie data (more accurate CLNF model) -model = 'model/main_clnf_general.txt'; -% Trained on in-the-wild -%model = 'model/main_clnf_wild.txt'; - -command = executable; - -command = cat(2, command, [' -fdir "' in_dir '"']); - -if(verbose) - command = cat(2, command, [' -ofdir "' out_dir '"']); - command = cat(2, command, [' -oidir "' out_dir '"']); -end - -command = cat(2, command, [' -mloc "', model, '"']); +% Load images (-fdir), output images and all the features (-out_dir), use a +% user specified model (-mloc), and visualize everything (-verbose) +command = sprintf('%s -fdir "%s" -out_dir "%s" -verbose -mloc "%s" ', executable, in_dir, out_dir, model); % Demonstrates the multi-hypothesis slow landmark detection (more accurate % when dealing with non-frontal faces and less accurate face detections) % Comment to skip this functionality -command = cat(2, command, ' -wild '); +command = cat(2, command, ' -wild -multi_view 1'); if(isunix) unix(command); diff --git a/matlab_runners/Demos/run_demo_video_multi.m b/matlab_runners/Demos/run_demo_video_multi.m index a181448..f64c248 100644 --- a/matlab_runners/Demos/run_demo_video_multi.m +++ b/matlab_runners/Demos/run_demo_video_multi.m @@ -1,3 +1,5 @@ +% A demo how to run a multi-face face tracker + clear; if(isunix) @@ -5,45 +7,28 @@ if(isunix) else executable = '"../../x64/Release/FaceLandmarkVidMulti.exe"'; end - -output = './demo_vid/'; - -if(~exist(output, 'file')) - mkdir(output) -end in_files = dir('../../samples/multi_face.avi'); -% some parameters -verbose = true; -% Trained on in the wild and multi-pie data (less accurate CLM model) -%model = 'model/main_clm_general.txt'; -% Trained on in-the-wild -%model = 'model/main_clm_wild.txt'; +model = 'model/main_clnf_general.txt'; % Trained on in the wild and multi-pie data (a CLNF model) -% Trained on in the wild and multi-pie data (more accurate CLNF model) -model = 'model/main_clnf_general.txt'; -% Trained on in-the-wild -%model = 'model/main_clnf_wild.txt'; +% Uncomment the below models if you want to try them +%model = 'model/main_clnf_wild.txt'; % Trained on in-the-wild data only + +%model = 'model/main_clm_general.txt'; % Trained on in the wild and multi-pie data (less accurate SVR/CLM model) +%model = 'model/main_clm_wild.txt'; % Trained on in-the-wild + +% Create a command that will run the tracker on set of videos and display the output +command = sprintf('%s -mloc "%s" ', executable, model); -command = executable; -command = cat(2, command, [' -mloc "', model, '"']); % add all videos to single argument list (so as not to load the model anew % for every video) for i=1:numel(in_files) - - inputFile = ['../../samples/', in_files(i).name]; - [~, name, ~] = fileparts(inputFile); - + inputFile = ['../../samples/', in_files(i).name]; command = cat(2, command, [' -f "' inputFile '" ']); - - if(verbose) - outputVideo = ['"' output name '.avi' '"']; - command = cat(2, command, [' -ov ' outputVideo]); - end - end +% Call the executable if(isunix) unix(command); else diff --git a/matlab_runners/Demos/run_demo_videos.m b/matlab_runners/Demos/run_demo_videos.m index 0a83a99..37e93fe 100644 --- a/matlab_runners/Demos/run_demo_videos.m +++ b/matlab_runners/Demos/run_demo_videos.m @@ -1,3 +1,5 @@ +% A demo how to run a single-face face tracker + clear if(isunix) @@ -5,46 +7,29 @@ if(isunix) else executable = '"../../x64/Release/FaceLandmarkVid.exe"'; end - -output = './demo_vid/'; - -if(~exist(output, 'file')) - mkdir(output) -end in_files = dir('../../samples/*.wmv'); in_files = cat(1, in_files, dir('../../samples/*.avi')); -% some parameters -verbose = true; -% Trained on in the wild and multi-pie data (less accurate SVR/CLM model) -%model = 'model/main_clm_general.txt'; -% Trained on in-the-wild -%model = 'model/main_clm_wild.txt'; +model = 'model/main_clnf_general.txt'; % Trained on in the wild and multi-pie data (a CLNF model) -% Trained on in the wild and multi-pie data (more accurate CLNF model) -model = 'model/main_clnf_general.txt'; -% Trained on in-the-wild -%model = 'model/main_clnf_wild.txt'; +% Uncomment the below models if you want to try them +%model = 'model/main_clnf_wild.txt'; % Trained on in-the-wild data only -command = executable; -command = cat(2, command, [' -mloc "', model, '"']); -% add all videos to single argument list (so as not to load the model anew -% for every video) +%model = 'model/main_clm_general.txt'; % Trained on in the wild and multi-pie data (less accurate SVR/CLM model) +%model = 'model/main_clm_wild.txt'; % Trained on in-the-wild + +% Create a command that will run the tracker on set of videos and display the output +command = sprintf('%s -mloc "%s" ', executable, model); + +% add all videos to single argument list by appending -f comments +% so as not to load the model anew for every video) for i=1:numel(in_files) - - inputFile = ['../../samples/', in_files(i).name]; - [~, name, ~] = fileparts(inputFile); - + inputFile = ['../../samples/', in_files(i).name]; command = cat(2, command, [' -f "' inputFile '" ']); - - if(verbose) - outputVideo = ['"' output name '.avi' '"']; - command = cat(2, command, [' -ov ' outputVideo]); - end - end +% Call the executable if(isunix) unix(command); else diff --git a/matlab_runners/Demos/run_test_img_seq.m b/matlab_runners/Demos/run_test_img_seq.m new file mode 100644 index 0000000..1367141 --- /dev/null +++ b/matlab_runners/Demos/run_test_img_seq.m @@ -0,0 +1,29 @@ +% A test script on image sequences, making sure grayscale and 16 bit +% sequences work + +clear + +% The location executable will depend on the OS +if(isunix) + executable = '"../../build/bin/FeatureExtraction"'; +else + executable = '"../../x64/Release/FeatureExtraction.exe"'; +end + +% Input file +in_dirs = {'../../samples/image_sequence',... + '../../samples/image_sequence_gray', ... + '../../samples/image_sequence_16bit'}; + +% Where to store the output +output_dir = './processed_features/'; + +for i=1:numel(in_dirs) + command = sprintf('%s -fdir "%s" -out_dir "%s" -verbose', executable, in_dirs{i}, output_dir); + + if(isunix) + unix(command); + else + dos(command); + end +end \ No newline at end of file diff --git a/matlab_runners/Feature Point Experiments/Run_OF_on_images.m b/matlab_runners/Feature Point Experiments/Run_OF_on_images.m index 3cca6f5..e45fde0 100644 --- a/matlab_runners/Feature Point Experiments/Run_OF_on_images.m +++ b/matlab_runners/Feature Point Experiments/Run_OF_on_images.m @@ -26,9 +26,9 @@ else end if(isunix) - command = '"../../build/bin/FaceLandmarkImg"'; + executable = '"../../build/bin/FaceLandmarkImg"'; else - command = '"../../x64/Release/FaceLandmarkImg.exe"'; + executable = '"../../x64/Release/FaceLandmarkImg.exe"'; end if(any(strcmp(varargin, 'model'))) @@ -44,31 +44,20 @@ else multi_view = 0; end -command = cat(2, command, [' -mloc ' model ' ']); -command = cat(2, command, [' -multi_view ' num2str(multi_view) ' ']); - -tic -parfor i=1:numel(dataset_dirs) +command = sprintf('%s -mloc %s -multi_view %s -2Dfp ', executable, model, num2str(multi_view)); - input_loc = ['-fdir "', dataset_dirs{i}, '" ']; - command_c = cat(2, command, input_loc); - - out_loc = ['-ofdir "', output_loc, '" ']; - command_c = cat(2, command_c, out_loc); - - if(verbose) - out_im_loc = ['-oidir "', output_loc, '" ']; - command_c = cat(2, command_c, out_im_loc); - end +tic +for i=1:numel(dataset_dirs) - command_c = cat(2, command_c, ' -wild '); + command_c = sprintf('%s -fdir "%s" -bboxdir "%s" -out_dir "%s" -wild ',... + command, dataset_dirs{i}, dataset_dirs{i}, output_loc); if(isunix) unix(command_c, '-echo'); else dos(command_c); end - + end toc @@ -80,7 +69,7 @@ dirs = {[database_root '/AFW/']; [database_root '/helen/testset/']; [database_root 'lfpw/testset/'];}; -landmark_dets = dir([output_loc '/*.pts']); +landmark_dets = dir([output_loc '/*.csv']); landmark_det_dir = [output_loc '/']; @@ -92,18 +81,26 @@ shapes = zeros(68,2,num_imgs); curr = 0; +% work out which columns in the csv file are relevant +tab = readtable([landmark_det_dir, landmark_dets(1).name]); +column_names = tab.Properties.VariableNames; +landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'x_')); +landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'y_')); + for i=1:numel(dirs) - gt_labels = dir([dirs{i}, '*.pts']); + gt_labels = dir([dirs{i}, '*.pts']); for g=1:numel(gt_labels) curr = curr+1; gt_landmarks = dlmread([dirs{i}, gt_labels(g).name], ' ', 'A4..B71'); - + [~, name, ~] = fileparts(gt_labels(g).name); % find the corresponding detection - landmark_det = dlmread([landmark_det_dir, gt_labels(g).name], ' ', 'A4..B71'); + all_params = dlmread([landmark_det_dir, name, '.csv'], ',', 1, 0); + + landmark_det = [all_params(landmark_inds_x); all_params(landmark_inds_y)]'; labels(:,:,curr) = gt_landmarks; @@ -126,8 +123,8 @@ if(size(shapes,2) == 66 && size(labels,2) == 68) shapes = shapes(inds_66,:,:); end -% Center the pixel -labels = labels - 0.5; +% Center the pixel, and convert to OCV format +labels = labels - 1.5; err_outline = compute_error(labels, shapes); diff --git a/matlab_runners/Feature Point Experiments/results/fps_yt.mat b/matlab_runners/Feature Point Experiments/results/fps_yt.mat index b4b90a5..a4f6035 100644 Binary files a/matlab_runners/Feature Point Experiments/results/fps_yt.mat and b/matlab_runners/Feature Point Experiments/results/fps_yt.mat differ diff --git a/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf b/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf index 7e88056..bb6cc81 100644 Binary files a/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf and b/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf differ diff --git a/matlab_runners/Feature Point Experiments/results/landmark_detections.mat b/matlab_runners/Feature Point Experiments/results/landmark_detections.mat index 826d2ce..8b5e179 100644 Binary files a/matlab_runners/Feature Point Experiments/results/landmark_detections.mat and b/matlab_runners/Feature Point Experiments/results/landmark_detections.mat differ diff --git a/matlab_runners/Feature Point Experiments/results/landmark_detections.txt b/matlab_runners/Feature Point Experiments/results/landmark_detections.txt index 8e69a0d..ac697cb 100644 --- a/matlab_runners/Feature Point Experiments/results/landmark_detections.txt +++ b/matlab_runners/Feature Point Experiments/results/landmark_detections.txt @@ -1,9 +1,9 @@ Type, mean, median -err clnf: 0.054348, 0.040034 -err clnf wild: 0.053107, 0.038525 -err svr: 0.070552, 0.050640 -err svr wild: 0.067452, 0.048706 -err clnf no out: 0.043081, 0.029782 -err clnf wild no out: 0.041386, 0.027463 -err svr no out: 0.058766, 0.038836 -err svr wild no out: 0.054020, 0.036252 +err clnf: 0.054473, 0.040001 +err clnf wild: 0.053173, 0.038511 +err svr: 0.070548, 0.050640 +err svr wild: 0.067535, 0.048691 +err clnf no out: 0.043240, 0.030040 +err clnf wild no out: 0.041460, 0.027516 +err svr no out: 0.058769, 0.038858 +err svr wild no out: 0.054168, 0.036253 diff --git a/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m b/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m index 40e0b4e..c9a25e3 100644 --- a/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m +++ b/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m @@ -9,40 +9,26 @@ elseif(exist('D:/Dropbox/Dropbox/AAM/test data/', 'file')) database_root = 'D:/Dropbox/Dropbox/AAM/test data/'; elseif(exist('F:/Dropbox/AAM/test data/', 'file')) database_root = 'F:/Dropbox/AAM/test data/'; -else +elseif(exist('/multicomp/datasets/300-W/', 'file')) database_root = '/multicomp/datasets/300-W/'; +elseif(exist('/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data/', 'file')) + database_root = '/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data/'; end %% Run using CLNF in the wild model out_clnf = [curr_dir '/out_wild_clnf_wild/']; -if(~exist(out_clnf, 'file')) - mkdir(out_clnf); -end - [err_clnf_wild, err_no_out_clnf_wild] = Run_OF_on_images(out_clnf, database_root, 'use_afw', 'use_lfpw', 'use_ibug', 'use_helen', 'verbose', 'model', 'model/main_clnf_wild.txt', 'multi_view', 1); %% Run using SVR model out_svr = [curr_dir '/out_wild_svr_wild/']; -if(~exist(out_svr, 'file')) - mkdir(out_svr); -end - [err_svr_wild, err_no_out_svr_wild] = Run_OF_on_images(out_svr, database_root, 'use_afw', 'use_lfpw', 'use_ibug', 'use_helen', 'verbose', 'model', 'model/main_clm_wild.txt', 'multi_view', 1); %% Run using general CLNF model out_clnf = [curr_dir '/out_wild_clnf/']; -if(~exist(out_clnf, 'file')) - mkdir(out_clnf); -end - [err_clnf, err_no_out_clnf] = Run_OF_on_images(out_clnf, database_root, 'use_afw', 'use_lfpw', 'use_ibug', 'use_helen', 'verbose', 'model', 'model/main_clnf_general.txt', 'multi_view', 1); %% Run using SVR model out_svr = [curr_dir '/out_wild_svr/']; -if(~exist(out_svr, 'file')) - mkdir(out_svr); -end - [err_svr, err_no_out_svr] = Run_OF_on_images(out_svr, database_root, 'use_afw', 'use_lfpw', 'use_ibug', 'use_helen', 'verbose', 'model', 'model/main_clm_general.txt', 'multi_view', 1); %% @@ -93,7 +79,7 @@ load('landmark_det_baselines/zhu_wild.mat'); load('out_wild_clnf_wild/res.mat'); for i=1:size(labels,3) - diffs = squeeze(sum(sum(bsxfun(@plus, labels_all(18:end,:,:)-0.5, - labels([18:60,62:64,66:end],:,i)),1),2)); + diffs = squeeze(sum(sum(bsxfun(@plus, labels_all(18:end,:,:)-1.5, - labels([18:60,62:64,66:end],:,i)),1),2)); inds_in_cpp = cat(1, inds_in_cpp, find(diffs == 0)); end diff --git a/matlab_runners/Feature Point Experiments/run_yt_dataset.m b/matlab_runners/Feature Point Experiments/run_yt_dataset.m index 8d3c29a..1630faf 100644 --- a/matlab_runners/Feature Point Experiments/run_yt_dataset.m +++ b/matlab_runners/Feature Point Experiments/run_yt_dataset.m @@ -18,6 +18,8 @@ elseif(exist('D:/Dropbox/Dropbox/AAM/test data/', 'file')) database_root = 'D:/Dropbox/Dropbox/AAM/test data/'; elseif(exist('F:/Dropbox/AAM/test data/', 'file')) database_root = 'F:/Dropbox/AAM/test data/'; +elseif(exist('/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data/', 'file')) + database_root = '/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data/'; else database_root = '/multicomp/datasets/'; end @@ -26,19 +28,14 @@ database_root = [database_root, '/ytceleb/']; in_vids = dir([database_root '/*.avi']); -command = executable; -command = cat(2, command, ' -no3Dfp -noMparams -noPose -noGaze -noAUs '); +command = sprintf('%s -2Dfp -out_dir "%s" ', executable, output); % add all videos to single argument list (so as not to load the model anew % for every video) for i=1:numel(in_vids) - [~, name, ~] = fileparts(in_vids(i).name); - - % where to output tracking results - outputFile_fp = [output name '_fp.txt']; in_file_name = [database_root, '/', in_vids(i).name]; - command = cat(2, command, [' -f "' in_file_name '" -of "' outputFile_fp '"']); + command = cat(2, command, [' -f "' in_file_name '" ']); end if(isunix) @@ -53,22 +50,14 @@ output = 'yt_features_clm/'; if(~exist(output, 'file')) mkdir(output) end - -command = executable; -command = cat(2, command, ' -mloc model/main_clm_general.txt '); -command = cat(2, command, ' -no3Dfp -noMparams -noPose -noGaze -noAUs '); + +command = sprintf('%s -2Dfp -out_dir "%s" -mloc model/main_clm_general.txt ', executable, output); % add all videos to single argument list (so as not to load the model anew % for every video) -for i=1:numel(in_vids) - - [~, name, ~] = fileparts(in_vids(i).name); - - % where to output tracking results - outputFile_fp = [output name '_fp.txt']; - in_file_name = [database_root, '/', in_vids(i).name]; - - command = cat(2, command, [' -f "' in_file_name '" -of "' outputFile_fp '"']); +for i=1:numel(in_vids) + in_file_name = [database_root, '/', in_vids(i).name]; + command = cat(2, command, [' -f "' in_file_name '"']); end if(isunix) @@ -80,7 +69,7 @@ end d_loc = 'yt_features/'; d_loc_clm = 'yt_features_clm/'; -files_yt = dir([d_loc, '/*.txt']); +files_yt = dir([d_loc, '/*.csv']); preds_all = []; preds_all_clm = []; gts_all = []; @@ -95,7 +84,7 @@ for i = 1:numel(files_yt) pred_landmarks(:,1,:) = xs'; pred_landmarks(:,2,:) = ys'; - pred_landmarks_clm = dlmread([d_loc_clm, files_yt(i).name], ',', 1, 0); + pred_landmarks_clm = dlmread([d_loc_clm, files_yt(i).name], ',', 1, 0); pred_landmarks_clm = pred_landmarks_clm(:,5:end); xs = pred_landmarks_clm(:, 1:end/2); @@ -104,7 +93,7 @@ for i = 1:numel(files_yt) pred_landmarks_clm(:,1,:) = xs'; pred_landmarks_clm(:,2,:) = ys'; - load([database_root, name(1:end-3), '.mat']); + load([database_root, name, '.mat']); preds_all = cat(3, preds_all, pred_landmarks); preds_all_clm = cat(3, preds_all_clm, pred_landmarks_clm); gts_all = cat(3, gts_all, labels); diff --git a/matlab_runners/Full_test_suite.m b/matlab_runners/Full_test_suite.m index 877fdc4..f97a74b 100644 --- a/matlab_runners/Full_test_suite.m +++ b/matlab_runners/Full_test_suite.m @@ -23,7 +23,7 @@ cd('../'); cd('Action Unit Experiments'); run_AU_prediction_Bosphorus assert(mean(cccs_reg) > 0.56); -assert(mean(f1s_class) > 0.46); +assert(mean(f1s_class) > 0.49); run_AU_prediction_BP4D assert(mean(ints_cccs) > 0.6); @@ -48,10 +48,14 @@ assert(median_error < 9.0) cd('../'); %% Demos +clear; +close all; cd('Demos'); run_demo_images; run_demo_videos; run_demo_video_multi; +run_demo_align_size; +run_test_img_seq; feature_extraction_demo_vid; feature_extraction_demo_img_seq; gaze_extraction_demo_vid; diff --git a/matlab_runners/Gaze Experiments/extract_mpii_gaze_test.m b/matlab_runners/Gaze Experiments/extract_mpii_gaze_test.m index 734d28d..ee2dd3c 100644 --- a/matlab_runners/Gaze Experiments/extract_mpii_gaze_test.m +++ b/matlab_runners/Gaze Experiments/extract_mpii_gaze_test.m @@ -29,22 +29,21 @@ else executable = '"../../x64/Release/FaceLandmarkImg.exe"'; end -command = sprintf('%s -fx 1028 -fy 1028 -gaze ', executable); +command = sprintf('%s -fx 1028 -fy 1028 ', executable); p_dirs = dir([database_root, 'p*']); parfor p=1:numel(p_dirs) tic - input_loc = ['-fdir "', [database_root, p_dirs(p).name], '" ']; - out_img_loc = ['-oidir "', [output, p_dirs(p).name], '" ']; - out_p_loc = ['-opdir "', [output, p_dirs(p).name], '" ']; - command_c = cat(2, command, input_loc, out_img_loc, out_p_loc); + input_loc = ['-gaze -fdir "', [database_root, p_dirs(p).name], '" ']; + out_img_loc = ['-out_dir "', [output, p_dirs(p).name], '" ']; + command_c = cat(2, command, input_loc, out_img_loc); if(isunix) unix(command_c, '-echo'); else dos(command_c); - end; + end end %% @@ -65,29 +64,36 @@ for p=1:numel(p_dirs) for i=1:size(filenames, 1) - fname = sprintf('%s/%s/%d_%d_%d_%d_%d_%d_%d_det_0.pose', output, p_dirs(p).name,... + fname = sprintf('%s/%s/%d_%d_%d_%d_%d_%d_%d.csv', output, p_dirs(p).name,... filenames(i,1), filenames(i,2), filenames(i,3), filenames(i,4),... filenames(i,5), filenames(i,6), filenames(i,7)); - try - A = dlmread(fname, ' ', 'A79..F79'); - valid = true; - catch - A = zeros(1,6); - A(1,3) = -1; - A(1,6) = -1; - valid = false; + + if(p==1 && i==1) + % First read in the column names + tab = readtable(fname); + column_names = tab.Properties.VariableNames; + + gaze_0_ids = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_0_')); + gaze_1_ids = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_1_')); + end + + if(exist(fname, 'file')) + all_params = dlmread(fname, ',', 1, 0); + else + all_params = []; + end + + % If there was a face detected + if(size(all_params,1)>0) + predictions_r(curr,:) = all_params(1,gaze_0_ids); + predictions_l(curr,:) = all_params(1,gaze_1_ids); + else + predictions_r(curr,:) = [0,0,-1]; + predictions_l(curr,:) = [0,0,-1]; end head_rot = headpose(i,1:3); - - predictions_r(curr,:) = A(1:3); - predictions_l(curr,:) = A(4:6); - - if(~valid) - predictions_r(curr,:) = [0,0,-1]; - predictions_l(curr,:) = [0,0,-1]; - end - + gt_r(curr,:) = data.right.gaze(i,:)'; gt_r(curr,:) = gt_r(curr,:) / norm(gt_r(curr,:)); gt_l(curr,:) = data.left.gaze(i,:)'; diff --git a/matlab_runners/Gaze Experiments/mpii_1500_errs.mat b/matlab_runners/Gaze Experiments/mpii_1500_errs.mat index 8fa2d5d..2f6b97e 100644 Binary files a/matlab_runners/Gaze Experiments/mpii_1500_errs.mat and b/matlab_runners/Gaze Experiments/mpii_1500_errs.mat differ diff --git a/matlab_runners/Head Pose Experiments/calcBUerror.m b/matlab_runners/Head Pose Experiments/calcBUerror.m index be3e0ed..b05b203 100644 --- a/matlab_runners/Head Pose Experiments/calcBUerror.m +++ b/matlab_runners/Head Pose Experiments/calcBUerror.m @@ -16,7 +16,7 @@ seq_ids = {}; for i = 1:numel(seqNames) - [frame t, rels, sc tx ty tz rx ry rz] = textread([resDir seqNames{i} '.txt'], '%f, %f, %f, %f, %f, %f, %f, %f, %f, %f', 'headerlines', 1); + [frame t, rels, sc tx ty tz rx ry rz] = textread([resDir seqNames{i} '.csv'], '%f, %f, %f, %f, %f, %f, %f, %f, %f, %f', 'headerlines', 1); posesGround = load ([gtDir seqNames{i} '.dat']); % the reliabilities of head pose diff --git a/matlab_runners/Head Pose Experiments/calcBiwiError.m b/matlab_runners/Head Pose Experiments/calcBiwiError.m index 3b6cd1c..e9b9e91 100644 --- a/matlab_runners/Head Pose Experiments/calcBiwiError.m +++ b/matlab_runners/Head Pose Experiments/calcBiwiError.m @@ -16,7 +16,7 @@ for i=1:numel(seqNames) posesGround = load ([gtDir '/' seqNames{i} '/groundTruthPose.txt']); - [frame t, rels, sc tx ty tz rx ry rz] = textread([resDir '/' seqNames{i} '.txt'], '%f, %f, %f, %f, %f, %f, %f, %f, %f, %f', 'headerlines', 1); + [frame t, rels, sc tx ty tz rx ry rz] = textread([resDir '/' seqNames{i} '.csv'], '%f, %f, %f, %f, %f, %f, %f, %f, %f, %f', 'headerlines', 1); % the reliabilities of head pose rels_all = cat(1, rels_all, rels); diff --git a/matlab_runners/Head Pose Experiments/calcIctError.m b/matlab_runners/Head Pose Experiments/calcIctError.m index b0df1a9..20a7161 100644 --- a/matlab_runners/Head Pose Experiments/calcIctError.m +++ b/matlab_runners/Head Pose Experiments/calcIctError.m @@ -4,7 +4,7 @@ function [meanError, all_rot_preds, all_rot_gts, meanErrors, all_errors, rels_al polhemus = 'polhemusNorm.csv'; - sequences = dir([resDir '*.txt']); + sequences = dir([resDir '*.csv']); rotMeanErr = zeros(numel(sequences),3); rotRMS = zeros(numel(sequences),3); diff --git a/matlab_runners/Head Pose Experiments/results/Pose_OF.mat b/matlab_runners/Head Pose Experiments/results/Pose_OF.mat index 8e6a34d..72ddab2 100644 Binary files a/matlab_runners/Head Pose Experiments/results/Pose_OF.mat and b/matlab_runners/Head Pose Experiments/results/Pose_OF.mat differ diff --git a/matlab_runners/Head Pose Experiments/results/Pose_OF.txt b/matlab_runners/Head Pose Experiments/results/Pose_OF.txt index 3d11618..3eafc97 100644 --- a/matlab_runners/Head Pose Experiments/results/Pose_OF.txt +++ b/matlab_runners/Head Pose Experiments/results/Pose_OF.txt @@ -1,4 +1,4 @@ Dataset and model, pitch, yaw, roll, mean, median -biwi error: 7.779, 6.302, 4.440, 6.174, 2.779 -bu error: 2.739, 3.348, 2.458, 2.848, 1.975 -ict error: 3.501, 3.988, 3.298, 3.596, 1.968 +biwi error: 7.778, 6.302, 4.441, 6.174, 2.777 +bu error: 2.739, 3.349, 2.459, 2.849, 1.974 +ict error: 3.498, 3.986, 3.298, 3.594, 1.966 diff --git a/matlab_runners/Head Pose Experiments/run_biwi_experiment.m b/matlab_runners/Head Pose Experiments/run_biwi_experiment.m index 8a564cf..5690a2f 100644 --- a/matlab_runners/Head Pose Experiments/run_biwi_experiment.m +++ b/matlab_runners/Head Pose Experiments/run_biwi_experiment.m @@ -10,50 +10,27 @@ end output_dir = 'experiments/biwi_out'; dbSeqDir = dir([rootDir biwiDir]); - +dbSeqDir = dbSeqDir(3:end); + output_dir = cat(2, output_dir, '/'); -offset = 0; - -r = 1 + offset; - -numTogether = 25; - - -for i=3 + offset:numTogether:numel(dbSeqDir) - - - command = executable; - - command = cat(2, command, [' -inroot ' '"' rootDir '"']); +command = sprintf('%s -inroot "%s" -out_dir "%s" -fx 505 -fy 505 -cx 320 -cy 240 -pose -vis-track ', executable, rootDir, output_dir); - % deal with edge cases - if(numTogether + i > numel(dbSeqDir)) - numTogether = numel(dbSeqDir) - i + 1; - end +if(verbose) + command = cat(2, command, [' -tracked ' outputVideo]); +end - for n=0:numTogether-1 - - inputFile = [biwiDir dbSeqDir(i+n).name '/colour.avi']; - outputFile = [output_dir dbSeqDir(i+n).name '.txt']; - - command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '"']); - - if(verbose) - outputVideo = [output_dir dbSeqDir(i).name '.avi']; - command = cat(2, command, [' -ov "' outputVideo '"']); - end - end - command = cat(2, command, [' -fx 505 -fy 505 -cx 320 -cy 240 -no2Dfp -no3Dfp -noMparams -noAUs -noGaze -vis-track ']); - - if(any(strcmp('model', varargin))) - command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']); - end - - r = r+1; - if(isunix) - unix(command, '-echo') - else - dos(command); - end +if(any(strcmp('model', varargin))) + command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']); end + +for i=1:numel(dbSeqDir) + inputFile = [biwiDir dbSeqDir(i).name '/colour.avi']; + command = sprintf('%s -f "%s" -of "%s" ', command, inputFile, dbSeqDir(i).name); +end + +if(isunix) + unix(command, '-echo') +else + dos(command); +end \ No newline at end of file diff --git a/matlab_runners/Head Pose Experiments/run_bu_experiment.m b/matlab_runners/Head Pose Experiments/run_bu_experiment.m index 2aa4bfa..a23125c 100644 --- a/matlab_runners/Head Pose Experiments/run_bu_experiment.m +++ b/matlab_runners/Head Pose Experiments/run_bu_experiment.m @@ -10,48 +10,27 @@ function [output_dir] = run_bu_experiment(bu_dir, verbose, varargin) buFiles = dir([bu_dir '*.avi']); - numTogether = 25; - - for i=1:numTogether:numel(buFiles) + % Only outputing the pose (-pose) + command = sprintf('%s -inroot "%s" -out_dir "%s" -fx 500 -fy 500 -cx 160 -cy 120 -pose -vis-track ', executable, bu_dir, output_dir); - command = executable; - command = cat(2, command, [' -inroot ' '"' bu_dir '/"']); - - % BU dataset orientation is in terms of camera plane, instruct the - % tracker to output it in that format - command = cat(2, command, [' -cp ']); - - % deal with edge cases - if(numTogether + i > numel(buFiles)) - numTogether = numel(buFiles) - i + 1; - end - - for n=0:numTogether-1 - inputFile = [buFiles(n+i).name]; - [~, name, ~] = fileparts(inputFile); - - % where to output results - outputFile = [output_dir name '.txt']; - - command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '"']); - - if(verbose) - outputVideo = ['"' output_dir name '.avi' '"']; - command = cat(2, command, [' -ov ' outputVideo]); - end - end - - command = cat(2, command, ' -fx 500 -fy 500 -cx 160 -cy 120 -no2Dfp -no3Dfp -noMparams -noAUs -noGaze -vis-track '); - - if(any(strcmp('model', varargin))) - command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']); - end - - if(isunix) - unix(command, '-echo') - else - dos(command); - end + for i=1:numel(buFiles) + inputFile = [buFiles(i).name]; + command = cat(2, command, sprintf(' -f "%s" ', inputFile)); end - + + + if(verbose) + command = cat(2, command, [' -tracked ' outputVideo]); + end + + if(any(strcmp('model', varargin))) + command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']); + end + + if(isunix) + unix(command, '-echo') + else + dos(command); + end + end \ No newline at end of file diff --git a/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m b/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m index f22ddce..bc12275 100644 --- a/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m +++ b/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m @@ -12,8 +12,10 @@ elseif(exist([getenv('USERPROFILE') 'F:/Dropbox/Dropbox/AAM/test data/'], 'file' database_root = 'F:/Dropbox/Dropbox/AAM/test data/'; elseif(exist('F:/Dropbox/AAM/test data/', 'file')) database_root = 'F:/Dropbox/AAM/test data/'; -else +elseif(exist('/multicomp/datasets/head_pose_dbs', 'file')) database_root = '/multicomp/datasets/head_pose_dbs/'; +elseif(exist('/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data', 'file')) + database_root = '/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data'; end buDir = [database_root, '/bu/uniform-light/']; diff --git a/matlab_runners/Head Pose Experiments/run_ict_experiment.m b/matlab_runners/Head Pose Experiments/run_ict_experiment.m index 75e2352..06b1f60 100644 --- a/matlab_runners/Head Pose Experiments/run_ict_experiment.m +++ b/matlab_runners/Head Pose Experiments/run_ict_experiment.m @@ -11,44 +11,29 @@ end output_dir = 'experiments/ict_out'; dbSeqDir = dir([rootDir ictDir]); - +dbSeqDir = dbSeqDir(3:end); + output_dir = cat(2, output_dir, '/'); -numTogether = 10; +command = sprintf('%s -inroot "%s" -out_dir "%s" -fx 535 -fy 536 -cx 327 -cy 241 -pose -vis-track ', executable, rootDir, output_dir); + +if(verbose) + command = cat(2, command, [' -tracked ' outputVideo]); +end -for i=3:numTogether:numel(dbSeqDir) - - command = [executable ' -fx 535 -fy 536 -cx 327 -cy 241 -no2Dfp -no3Dfp -noMparams -noAUs -noGaze -vis-track ']; +if(any(strcmp('model', varargin))) + command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']); +end - command = cat(2, command, [' -inroot ' '"' rootDir '/"']); - - % deal with edge cases - if(numTogether + i > numel(dbSeqDir)) - numTogether = numel(dbSeqDir) - i + 1; - end - - for n=0:numTogether-1 - - inputFile = [ictDir dbSeqDir(i+n).name '/colour undist.avi']; - outputFile = [output_dir dbSeqDir(i+n).name '.txt']; - - command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '" ']); - - if(verbose) - outputVideo = [output_dir dbSeqDir(i+n).name '.avi']; - command = cat(2, command, [' -ov "' outputVideo '"']); - end - end - - if(any(strcmp('model', varargin))) - command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']); - end - - if(isunix) - unix(command, '-echo') - else - dos(command); - end +for i=1:numel(dbSeqDir) + inputFile = [ictDir dbSeqDir(i).name '/colour undist.avi']; + command = cat(2, command, [' -f "' inputFile '" -of "' dbSeqDir(i).name '" ']); +end + +if(isunix) + unix(command, '-echo') +else + dos(command); end diff --git a/matlab_version/face_detection/mtcnn/test1.jpg b/matlab_version/face_detection/mtcnn/test1.jpg new file mode 100644 index 0000000..41c7563 Binary files /dev/null and b/matlab_version/face_detection/mtcnn/test1.jpg differ diff --git a/image_sequence/001.jpg b/samples/image_sequence/001.jpg similarity index 100% rename from image_sequence/001.jpg rename to samples/image_sequence/001.jpg diff --git a/image_sequence/002.jpg b/samples/image_sequence/002.jpg similarity index 100% rename from image_sequence/002.jpg rename to samples/image_sequence/002.jpg diff --git a/image_sequence/003.jpg b/samples/image_sequence/003.jpg similarity index 100% rename from image_sequence/003.jpg rename to samples/image_sequence/003.jpg diff --git a/image_sequence/004.jpg b/samples/image_sequence/004.jpg similarity index 100% rename from image_sequence/004.jpg rename to samples/image_sequence/004.jpg diff --git a/image_sequence/005.jpg b/samples/image_sequence/005.jpg similarity index 100% rename from image_sequence/005.jpg rename to samples/image_sequence/005.jpg diff --git a/image_sequence/006.jpg b/samples/image_sequence/006.jpg similarity index 100% rename from image_sequence/006.jpg rename to samples/image_sequence/006.jpg diff --git a/image_sequence/007.jpg b/samples/image_sequence/007.jpg similarity index 100% rename from image_sequence/007.jpg rename to samples/image_sequence/007.jpg diff --git a/image_sequence/008.jpg b/samples/image_sequence/008.jpg similarity index 100% rename from image_sequence/008.jpg rename to samples/image_sequence/008.jpg diff --git a/image_sequence/009.jpg b/samples/image_sequence/009.jpg similarity index 100% rename from image_sequence/009.jpg rename to samples/image_sequence/009.jpg diff --git a/image_sequence/010.jpg b/samples/image_sequence/010.jpg similarity index 100% rename from image_sequence/010.jpg rename to samples/image_sequence/010.jpg diff --git a/image_sequence/011.jpg b/samples/image_sequence/011.jpg similarity index 100% rename from image_sequence/011.jpg rename to samples/image_sequence/011.jpg diff --git a/image_sequence/012.jpg b/samples/image_sequence/012.jpg similarity index 100% rename from image_sequence/012.jpg rename to samples/image_sequence/012.jpg diff --git a/image_sequence/013.jpg b/samples/image_sequence/013.jpg similarity index 100% rename from image_sequence/013.jpg rename to samples/image_sequence/013.jpg diff --git a/image_sequence/014.jpg b/samples/image_sequence/014.jpg similarity index 100% rename from image_sequence/014.jpg rename to samples/image_sequence/014.jpg diff --git a/image_sequence/015.jpg b/samples/image_sequence/015.jpg similarity index 100% rename from image_sequence/015.jpg rename to samples/image_sequence/015.jpg diff --git a/image_sequence/016.jpg b/samples/image_sequence/016.jpg similarity index 100% rename from image_sequence/016.jpg rename to samples/image_sequence/016.jpg diff --git a/image_sequence/017.jpg b/samples/image_sequence/017.jpg similarity index 100% rename from image_sequence/017.jpg rename to samples/image_sequence/017.jpg diff --git a/image_sequence/018.jpg b/samples/image_sequence/018.jpg similarity index 100% rename from image_sequence/018.jpg rename to samples/image_sequence/018.jpg diff --git a/image_sequence/019.jpg b/samples/image_sequence/019.jpg similarity index 100% rename from image_sequence/019.jpg rename to samples/image_sequence/019.jpg diff --git a/image_sequence/020.jpg b/samples/image_sequence/020.jpg similarity index 100% rename from image_sequence/020.jpg rename to samples/image_sequence/020.jpg diff --git a/image_sequence/021.jpg b/samples/image_sequence/021.jpg similarity index 100% rename from image_sequence/021.jpg rename to samples/image_sequence/021.jpg diff --git a/image_sequence/022.jpg b/samples/image_sequence/022.jpg similarity index 100% rename from image_sequence/022.jpg rename to samples/image_sequence/022.jpg diff --git a/image_sequence/023.jpg b/samples/image_sequence/023.jpg similarity index 100% rename from image_sequence/023.jpg rename to samples/image_sequence/023.jpg diff --git a/image_sequence/024.jpg b/samples/image_sequence/024.jpg similarity index 100% rename from image_sequence/024.jpg rename to samples/image_sequence/024.jpg diff --git a/image_sequence/025.jpg b/samples/image_sequence/025.jpg similarity index 100% rename from image_sequence/025.jpg rename to samples/image_sequence/025.jpg diff --git a/image_sequence/026.jpg b/samples/image_sequence/026.jpg similarity index 100% rename from image_sequence/026.jpg rename to samples/image_sequence/026.jpg diff --git a/image_sequence/027.jpg b/samples/image_sequence/027.jpg similarity index 100% rename from image_sequence/027.jpg rename to samples/image_sequence/027.jpg diff --git a/image_sequence/028.jpg b/samples/image_sequence/028.jpg similarity index 100% rename from image_sequence/028.jpg rename to samples/image_sequence/028.jpg diff --git a/image_sequence/029.jpg b/samples/image_sequence/029.jpg similarity index 100% rename from image_sequence/029.jpg rename to samples/image_sequence/029.jpg diff --git a/image_sequence/030.jpg b/samples/image_sequence/030.jpg similarity index 100% rename from image_sequence/030.jpg rename to samples/image_sequence/030.jpg diff --git a/samples/image_sequence_16bit/001.png b/samples/image_sequence_16bit/001.png new file mode 100644 index 0000000..6d97f97 Binary files /dev/null and b/samples/image_sequence_16bit/001.png differ diff --git a/samples/image_sequence_16bit/002.png b/samples/image_sequence_16bit/002.png new file mode 100644 index 0000000..34d4ee3 Binary files /dev/null and b/samples/image_sequence_16bit/002.png differ diff --git a/samples/image_sequence_16bit/003.png b/samples/image_sequence_16bit/003.png new file mode 100644 index 0000000..7657c39 Binary files /dev/null and b/samples/image_sequence_16bit/003.png differ diff --git a/samples/image_sequence_16bit/004.png b/samples/image_sequence_16bit/004.png new file mode 100644 index 0000000..c094507 Binary files /dev/null and b/samples/image_sequence_16bit/004.png differ diff --git a/samples/image_sequence_16bit/005.png b/samples/image_sequence_16bit/005.png new file mode 100644 index 0000000..453acb0 Binary files /dev/null and b/samples/image_sequence_16bit/005.png differ diff --git a/samples/image_sequence_16bit/006.png b/samples/image_sequence_16bit/006.png new file mode 100644 index 0000000..746f351 Binary files /dev/null and b/samples/image_sequence_16bit/006.png differ diff --git a/samples/image_sequence_16bit/007.png b/samples/image_sequence_16bit/007.png new file mode 100644 index 0000000..908a7e5 Binary files /dev/null and b/samples/image_sequence_16bit/007.png differ diff --git a/samples/image_sequence_16bit/008.png b/samples/image_sequence_16bit/008.png new file mode 100644 index 0000000..1163503 Binary files /dev/null and b/samples/image_sequence_16bit/008.png differ diff --git a/samples/image_sequence_16bit/009.png b/samples/image_sequence_16bit/009.png new file mode 100644 index 0000000..9c63cdf Binary files /dev/null and b/samples/image_sequence_16bit/009.png differ diff --git a/samples/image_sequence_16bit/010.png b/samples/image_sequence_16bit/010.png new file mode 100644 index 0000000..642a622 Binary files /dev/null and b/samples/image_sequence_16bit/010.png differ diff --git a/samples/image_sequence_16bit/011.png b/samples/image_sequence_16bit/011.png new file mode 100644 index 0000000..976ddf0 Binary files /dev/null and b/samples/image_sequence_16bit/011.png differ diff --git a/samples/image_sequence_16bit/012.png b/samples/image_sequence_16bit/012.png new file mode 100644 index 0000000..cdb5a6b Binary files /dev/null and b/samples/image_sequence_16bit/012.png differ diff --git a/samples/image_sequence_16bit/013.png b/samples/image_sequence_16bit/013.png new file mode 100644 index 0000000..60ac230 Binary files /dev/null and b/samples/image_sequence_16bit/013.png differ diff --git a/samples/image_sequence_16bit/014.png b/samples/image_sequence_16bit/014.png new file mode 100644 index 0000000..18469be Binary files /dev/null and b/samples/image_sequence_16bit/014.png differ diff --git a/samples/image_sequence_16bit/015.png b/samples/image_sequence_16bit/015.png new file mode 100644 index 0000000..9a6dd9f Binary files /dev/null and b/samples/image_sequence_16bit/015.png differ diff --git a/samples/image_sequence_16bit/016.png b/samples/image_sequence_16bit/016.png new file mode 100644 index 0000000..7e115ff Binary files /dev/null and b/samples/image_sequence_16bit/016.png differ diff --git a/samples/image_sequence_16bit/017.png b/samples/image_sequence_16bit/017.png new file mode 100644 index 0000000..7972faa Binary files /dev/null and b/samples/image_sequence_16bit/017.png differ diff --git a/samples/image_sequence_16bit/018.png b/samples/image_sequence_16bit/018.png new file mode 100644 index 0000000..ede13b2 Binary files /dev/null and b/samples/image_sequence_16bit/018.png differ diff --git a/samples/image_sequence_16bit/019.png b/samples/image_sequence_16bit/019.png new file mode 100644 index 0000000..7b85e7a Binary files /dev/null and b/samples/image_sequence_16bit/019.png differ diff --git a/samples/image_sequence_16bit/020.png b/samples/image_sequence_16bit/020.png new file mode 100644 index 0000000..f141bea Binary files /dev/null and b/samples/image_sequence_16bit/020.png differ diff --git a/samples/image_sequence_16bit/021.png b/samples/image_sequence_16bit/021.png new file mode 100644 index 0000000..7d353a1 Binary files /dev/null and b/samples/image_sequence_16bit/021.png differ diff --git a/samples/image_sequence_16bit/022.png b/samples/image_sequence_16bit/022.png new file mode 100644 index 0000000..fbb22de Binary files /dev/null and b/samples/image_sequence_16bit/022.png differ diff --git a/samples/image_sequence_16bit/023.png b/samples/image_sequence_16bit/023.png new file mode 100644 index 0000000..f0f7be0 Binary files /dev/null and b/samples/image_sequence_16bit/023.png differ diff --git a/samples/image_sequence_16bit/024.png b/samples/image_sequence_16bit/024.png new file mode 100644 index 0000000..992baf7 Binary files /dev/null and b/samples/image_sequence_16bit/024.png differ diff --git a/samples/image_sequence_16bit/025.png b/samples/image_sequence_16bit/025.png new file mode 100644 index 0000000..e678b58 Binary files /dev/null and b/samples/image_sequence_16bit/025.png differ diff --git a/samples/image_sequence_16bit/026.png b/samples/image_sequence_16bit/026.png new file mode 100644 index 0000000..1a9868e Binary files /dev/null and b/samples/image_sequence_16bit/026.png differ diff --git a/samples/image_sequence_16bit/027.png b/samples/image_sequence_16bit/027.png new file mode 100644 index 0000000..053f1df Binary files /dev/null and b/samples/image_sequence_16bit/027.png differ diff --git a/samples/image_sequence_16bit/028.png b/samples/image_sequence_16bit/028.png new file mode 100644 index 0000000..297d91f Binary files /dev/null and b/samples/image_sequence_16bit/028.png differ diff --git a/samples/image_sequence_16bit/029.png b/samples/image_sequence_16bit/029.png new file mode 100644 index 0000000..ba26343 Binary files /dev/null and b/samples/image_sequence_16bit/029.png differ diff --git a/samples/image_sequence_16bit/030.png b/samples/image_sequence_16bit/030.png new file mode 100644 index 0000000..b7c5723 Binary files /dev/null and b/samples/image_sequence_16bit/030.png differ diff --git a/samples/image_sequence_gray/001.jpg b/samples/image_sequence_gray/001.jpg new file mode 100644 index 0000000..ecd6441 Binary files /dev/null and b/samples/image_sequence_gray/001.jpg differ diff --git a/samples/image_sequence_gray/002.jpg b/samples/image_sequence_gray/002.jpg new file mode 100644 index 0000000..f9ff69c Binary files /dev/null and b/samples/image_sequence_gray/002.jpg differ diff --git a/samples/image_sequence_gray/003.jpg b/samples/image_sequence_gray/003.jpg new file mode 100644 index 0000000..7aa3f70 Binary files /dev/null and b/samples/image_sequence_gray/003.jpg differ diff --git a/samples/image_sequence_gray/004.jpg b/samples/image_sequence_gray/004.jpg new file mode 100644 index 0000000..c15b9b2 Binary files /dev/null and b/samples/image_sequence_gray/004.jpg differ diff --git a/samples/image_sequence_gray/005.jpg b/samples/image_sequence_gray/005.jpg new file mode 100644 index 0000000..59c0b93 Binary files /dev/null and b/samples/image_sequence_gray/005.jpg differ diff --git a/samples/image_sequence_gray/006.jpg b/samples/image_sequence_gray/006.jpg new file mode 100644 index 0000000..9f6adae Binary files /dev/null and b/samples/image_sequence_gray/006.jpg differ diff --git a/samples/image_sequence_gray/007.jpg b/samples/image_sequence_gray/007.jpg new file mode 100644 index 0000000..04c0d37 Binary files /dev/null and b/samples/image_sequence_gray/007.jpg differ diff --git a/samples/image_sequence_gray/008.jpg b/samples/image_sequence_gray/008.jpg new file mode 100644 index 0000000..b7bcc85 Binary files /dev/null and b/samples/image_sequence_gray/008.jpg differ diff --git a/samples/image_sequence_gray/009.jpg b/samples/image_sequence_gray/009.jpg new file mode 100644 index 0000000..adb2700 Binary files /dev/null and b/samples/image_sequence_gray/009.jpg differ diff --git a/samples/image_sequence_gray/010.jpg b/samples/image_sequence_gray/010.jpg new file mode 100644 index 0000000..b4c855b Binary files /dev/null and b/samples/image_sequence_gray/010.jpg differ diff --git a/samples/image_sequence_gray/011.jpg b/samples/image_sequence_gray/011.jpg new file mode 100644 index 0000000..bce0460 Binary files /dev/null and b/samples/image_sequence_gray/011.jpg differ diff --git a/samples/image_sequence_gray/012.jpg b/samples/image_sequence_gray/012.jpg new file mode 100644 index 0000000..0318691 Binary files /dev/null and b/samples/image_sequence_gray/012.jpg differ diff --git a/samples/image_sequence_gray/013.jpg b/samples/image_sequence_gray/013.jpg new file mode 100644 index 0000000..354302d Binary files /dev/null and b/samples/image_sequence_gray/013.jpg differ diff --git a/samples/image_sequence_gray/014.jpg b/samples/image_sequence_gray/014.jpg new file mode 100644 index 0000000..8c2e2e3 Binary files /dev/null and b/samples/image_sequence_gray/014.jpg differ diff --git a/samples/image_sequence_gray/015.jpg b/samples/image_sequence_gray/015.jpg new file mode 100644 index 0000000..fd319c4 Binary files /dev/null and b/samples/image_sequence_gray/015.jpg differ diff --git a/samples/image_sequence_gray/016.jpg b/samples/image_sequence_gray/016.jpg new file mode 100644 index 0000000..06c1559 Binary files /dev/null and b/samples/image_sequence_gray/016.jpg differ diff --git a/samples/image_sequence_gray/017.jpg b/samples/image_sequence_gray/017.jpg new file mode 100644 index 0000000..ea0e86b Binary files /dev/null and b/samples/image_sequence_gray/017.jpg differ diff --git a/samples/image_sequence_gray/018.jpg b/samples/image_sequence_gray/018.jpg new file mode 100644 index 0000000..6afd389 Binary files /dev/null and b/samples/image_sequence_gray/018.jpg differ diff --git a/samples/image_sequence_gray/019.jpg b/samples/image_sequence_gray/019.jpg new file mode 100644 index 0000000..0385666 Binary files /dev/null and b/samples/image_sequence_gray/019.jpg differ diff --git a/samples/image_sequence_gray/020.jpg b/samples/image_sequence_gray/020.jpg new file mode 100644 index 0000000..0b88047 Binary files /dev/null and b/samples/image_sequence_gray/020.jpg differ diff --git a/samples/image_sequence_gray/021.jpg b/samples/image_sequence_gray/021.jpg new file mode 100644 index 0000000..709c0d2 Binary files /dev/null and b/samples/image_sequence_gray/021.jpg differ diff --git a/samples/image_sequence_gray/022.jpg b/samples/image_sequence_gray/022.jpg new file mode 100644 index 0000000..d4b58a9 Binary files /dev/null and b/samples/image_sequence_gray/022.jpg differ diff --git a/samples/image_sequence_gray/023.jpg b/samples/image_sequence_gray/023.jpg new file mode 100644 index 0000000..0193838 Binary files /dev/null and b/samples/image_sequence_gray/023.jpg differ diff --git a/samples/image_sequence_gray/024.jpg b/samples/image_sequence_gray/024.jpg new file mode 100644 index 0000000..df5902e Binary files /dev/null and b/samples/image_sequence_gray/024.jpg differ diff --git a/samples/image_sequence_gray/025.jpg b/samples/image_sequence_gray/025.jpg new file mode 100644 index 0000000..b5ef150 Binary files /dev/null and b/samples/image_sequence_gray/025.jpg differ diff --git a/samples/image_sequence_gray/026.jpg b/samples/image_sequence_gray/026.jpg new file mode 100644 index 0000000..e3bb771 Binary files /dev/null and b/samples/image_sequence_gray/026.jpg differ diff --git a/samples/image_sequence_gray/027.jpg b/samples/image_sequence_gray/027.jpg new file mode 100644 index 0000000..d073674 Binary files /dev/null and b/samples/image_sequence_gray/027.jpg differ diff --git a/samples/image_sequence_gray/028.jpg b/samples/image_sequence_gray/028.jpg new file mode 100644 index 0000000..85ed2c9 Binary files /dev/null and b/samples/image_sequence_gray/028.jpg differ diff --git a/samples/image_sequence_gray/029.jpg b/samples/image_sequence_gray/029.jpg new file mode 100644 index 0000000..5f572cb Binary files /dev/null and b/samples/image_sequence_gray/029.jpg differ diff --git a/samples/image_sequence_gray/030.jpg b/samples/image_sequence_gray/030.jpg new file mode 100644 index 0000000..702f710 Binary files /dev/null and b/samples/image_sequence_gray/030.jpg differ diff --git a/samples/sample3_16bit_gray.png b/samples/sample3_16bit_gray.png new file mode 100644 index 0000000..32cad7f Binary files /dev/null and b/samples/sample3_16bit_gray.png differ diff --git a/samples/sample3_16bit_rgb.png b/samples/sample3_16bit_rgb.png new file mode 100644 index 0000000..15d4011 Binary files /dev/null and b/samples/sample3_16bit_rgb.png differ diff --git a/samples/sample4.jpg b/samples/sample4.jpg index 35523d8..fb0c5b6 100644 Binary files a/samples/sample4.jpg and b/samples/sample4.jpg differ