diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index 4b0336f..6133cde 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -73,182 +73,6 @@ vector get_arguments(int argc, char **argv) } // TODO rem -void convert_to_grayscale(const cv::Mat& in, cv::Mat& out) -{ - if(in.channels() == 3) - { - // Make sure it's in a correct format - if(in.depth() != CV_8U) - { - if(in.depth() == CV_16U) - { - cv::Mat tmp = in / 256; - tmp.convertTo(tmp, CV_8U); - cv::cvtColor(tmp, out, CV_BGR2GRAY); - } - } - else - { - cv::cvtColor(in, out, CV_BGR2GRAY); - } - } - else if(in.channels() == 4) - { - cv::cvtColor(in, out, CV_BGRA2GRAY); - } - else - { - if(in.depth() == CV_16U) - { - cv::Mat tmp = in / 256; - out = tmp.clone(); - } - else if(in.depth() != CV_8U) - { - in.convertTo(out, CV_8U); - } - else - { - out = in.clone(); - } - } -} - -// Useful utility for creating directories for storing the output files -void create_directory_from_file(string output_path) -{ - - // Creating the right directory structure - - // First get rid of the file - auto p = boost::filesystem::path(boost::filesystem::path(output_path).parent_path()); - - if (!p.empty() && !boost::filesystem::exists(p)) - { - bool success = boost::filesystem::create_directories(p); - if (!success) - { - cout << "Failed to create a directory... " << p.string() << endl; - } - } -} - -// This will only be accurate when camera parameters are accurate, useful for work on 3D data -void write_out_pose_landmarks(const string& outfeatures, const cv::Mat_& shape3D, const cv::Vec6d& pose, const cv::Point3f& gaze0, const cv::Point3f& gaze1) -{ - create_directory_from_file(outfeatures); - std::ofstream featuresFile; - featuresFile.open(outfeatures); - - if (featuresFile.is_open()) - { - int n = shape3D.cols; - featuresFile << "version: 1" << endl; - featuresFile << "npoints: " << n << endl; - featuresFile << "{" << endl; - - for (int i = 0; i < n; ++i) - { - // Use matlab format, so + 1 - featuresFile << shape3D.at(i) << " " << shape3D.at(i + n) << " " << shape3D.at(i + 2*n) << endl; - } - featuresFile << "}" << endl; - - // Do the pose and eye gaze if present as well - featuresFile << "pose: eul_x, eul_y, eul_z: " << endl; - featuresFile << "{" << endl; - featuresFile << pose[3] << " " << pose[4] << " " << pose[5] << endl; - featuresFile << "}" << endl; - - // Do the pose and eye gaze if present as well - featuresFile << "gaze_vec: dir_x_1, dir_y_1, dir_z_1, dir_x_2, dir_y_2, dir_z_2: " << endl; - featuresFile << "{" << endl; - featuresFile << gaze0.x << " " << gaze0.y << " " << gaze0.z << " " << gaze1.x << " " << gaze1.y << " " << gaze1.z << endl; - featuresFile << "}" << endl; - - featuresFile.close(); - } -} - -void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF& clnf_model, const cv::Vec6d& pose, const cv::Point3f& gaze0, const cv::Point3f& gaze1, const cv::Vec2d gaze_angle, std::vector> au_intensities, std::vector> au_occurences, bool output_gaze) -{ - create_directory_from_file(outfeatures); - std::ofstream featuresFile; - featuresFile.open(outfeatures); - - if (featuresFile.is_open()) - { - int n = clnf_model.patch_experts.visibilities[0][0].rows; - featuresFile << "version: 2" << endl; - featuresFile << "npoints: " << n << endl; - featuresFile << "{" << endl; - - for (int i = 0; i < n; ++i) - { - // Use matlab format, so + 1 - featuresFile << clnf_model.detected_landmarks.at(i) + 1 << " " << clnf_model.detected_landmarks.at(i + n) + 1 << endl; - } - featuresFile << "}" << endl; - - // Do the pose and eye gaze if present as well - featuresFile << "pose: eul_x, eul_y, eul_z: " << endl; - featuresFile << "{" << endl; - featuresFile << pose[3] << " " << pose[4] << " " << pose[5] << endl; - featuresFile << "}" << endl; - - if(output_gaze) - { - featuresFile << "gaze: dir_x_1, dir_y_1, dir_z_1, dir_x_2, dir_y_2, dir_z_2: " << endl; - featuresFile << "{" << endl; - featuresFile << gaze0.x << " " << gaze0.y << " " << gaze0.z << " " << gaze1.x << " " << gaze1.y << " " << gaze1.z << endl; - featuresFile << "}" << endl; - - featuresFile << "gaze: angle_x, angle_y: " << endl; - featuresFile << "{" << endl; - featuresFile << gaze_angle[0] << " " << gaze_angle[1] << endl; - featuresFile << "}" << endl; - - std::vector eye_landmark_points = LandmarkDetector::CalculateAllEyeLandmarks(clnf_model); - - featuresFile << "eye_lmks: " << eye_landmark_points.size() << endl; - featuresFile << "{" << endl; - - for (int i = 0; i < eye_landmark_points.size(); ++i) - { - // Use matlab format, so + 1 - featuresFile << (eye_landmark_points[i].x + 1) << " " << (eye_landmark_points[i].y + 1) << endl; - } - featuresFile << "}" << endl; - } - // Do the au intensities - featuresFile << "au intensities: " << au_intensities.size() << endl; - featuresFile << "{" << endl; - - for (size_t i = 0; i < au_intensities.size(); ++i) - { - // Use matlab format, so + 1 - featuresFile << au_intensities[i].first << " " << au_intensities[i].second << endl; - } - - featuresFile << "}" << endl; - - // Do the au occurences - featuresFile << "au occurences: " << au_occurences.size() << endl; - featuresFile << "{" << endl; - - for (size_t i = 0; i < au_occurences.size(); ++i) - { - // Use matlab format, so + 1 - featuresFile << au_occurences[i].first << " " << au_occurences[i].second << endl; - } - - featuresFile << "}" << endl; - - - featuresFile.close(); - } -} - void create_display_image(const cv::Mat& orig, cv::Mat& display_image, LandmarkDetector::CLNF& clnf_model) { diff --git a/lib/local/Utilities/Utilities.vcxproj b/lib/local/Utilities/Utilities.vcxproj index 843f8ee..ded345a 100644 --- a/lib/local/Utilities/Utilities.vcxproj +++ b/lib/local/Utilities/Utilities.vcxproj @@ -148,6 +148,7 @@ + @@ -157,6 +158,7 @@ + diff --git a/lib/local/Utilities/Utilities.vcxproj.filters b/lib/local/Utilities/Utilities.vcxproj.filters index 28734b6..65b28a5 100644 --- a/lib/local/Utilities/Utilities.vcxproj.filters +++ b/lib/local/Utilities/Utilities.vcxproj.filters @@ -36,6 +36,9 @@ Source Files + + Source Files + @@ -62,5 +65,8 @@ Header Files + + Header Files + \ No newline at end of file diff --git a/lib/local/Utilities/include/ImageCapture.h b/lib/local/Utilities/include/ImageCapture.h new file mode 100644 index 0000000..9d239a9 --- /dev/null +++ b/lib/local/Utilities/include/ImageCapture.h @@ -0,0 +1,110 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __IMAGE_CAPTURE_h_ +#define __IMAGE_CAPTURE_h_ + +// System includes +#include +#include +#include + +// OpenCV includes +#include +#include + +namespace Utilities +{ + + //=========================================================================== + /** + A class for capturing sequences from video, webcam, and image directories + */ + class ImageCapture { + + public: + + // Default constructor + ImageCapture() {}; + + // TODO block copy, move etc. + + // Opening based on command line arguments + bool Open(std::vector& arguments); + + // Direct opening + + // Image sequence in the directory + bool OpenDirectory(std::string directory, float fx = -1, float fy = -1, float cx = -1, float cy = -1); + + // Video file + bool OpenImageFiles(const std::vector& image_files, float fx = -1, float fy = -1, float cx = -1, float cy = -1); + + // Getting the next frame + cv::Mat GetNextImage(); + + // Getting the most recent grayscale frame (need to call GetNextImage first) + cv::Mat_ GetGrayFrame(); + + // Parameters describing the sequence and it's progress (what's the proportion of images opened) + double GetProgress(); + + int image_width; + int image_height; + + float fx, fy, cx, cy; + + // Name of the video file, image directory, or the webcam + std::string name; + + private: + + // Storing the latest captures + cv::Mat latest_frame; + cv::Mat latest_gray_frame; + + // Keeping track of how many files are read and the filenames + size_t frame_num; + std::vector image_files; + + void SetCameraIntrinsics(float fx, float fy, float cx, float cy); + + // TODO make sure that can set fx, fy externally + bool image_intrinsics_set; + + // TODO make sure the error message makes sense + bool no_input_specified; + + }; +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h index 769b2e2..1b42143 100644 --- a/lib/local/Utilities/include/SequenceCapture.h +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -120,8 +120,6 @@ namespace Utilities // Length of video allowing to assess progress size_t vid_length; - bool img_grabbed; - // If using a webcam, helps to keep track of time int64 start_time; diff --git a/lib/local/Utilities/src/ImageCapture.cpp b/lib/local/Utilities/src/ImageCapture.cpp new file mode 100644 index 0000000..78920e9 --- /dev/null +++ b/lib/local/Utilities/src/ImageCapture.cpp @@ -0,0 +1,355 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "ImageCapture.h" + +#include + +// Boost includes +#include +#include +#include + +// OpenCV includes +#include + +using namespace Utilities; + +#define INFO_STREAM( stream ) \ +std::cout << stream << std::endl + +#define WARN_STREAM( stream ) \ +std::cout << "Warning: " << stream << std::endl + +#define ERROR_STREAM( stream ) \ +std::cout << "Error: " << stream << std::endl + +bool ImageCapture::Open(std::vector& arguments) +{ + + // Consuming the input arguments + bool* valid = new bool[arguments.size()]; + + for (size_t i = 0; i < arguments.size(); ++i) + { + valid[i] = true; + } + + // Some default values + std::string input_root = ""; + fx = -1; fy = -1; cx = -1; cy = -1; + frame_num = 0; + + std::string separator = std::string(1, boost::filesystem::path::preferred_separator); + + // First check if there is a root argument (so that videos and input directories could be defined more easily) + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-root") == 0) + { + input_root = arguments[i + 1] + separator; + i++; + } + if (arguments[i].compare("-inroot") == 0) + { + input_root = arguments[i + 1] + separator; + i++; + } + } + + std::string input_directory; + + bool directory_found = false; + + std::vector input_image_files; + + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-f") == 0) + { + input_image_files.push_back(input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-fdir") == 0) + { + if (directory_found) + { + WARN_STREAM("Input directory already found, using the first one:" + input_directory); + } + else + { + input_directory = (input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + directory_found = true; + } + } + else if (arguments[i].compare("-fx") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> fx; + i++; + } + else if (arguments[i].compare("-fy") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> fy; + i++; + } + else if (arguments[i].compare("-cx") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> cx; + i++; + } + else if (arguments[i].compare("-cy") == 0) + { + std::stringstream data(arguments[i + 1]); + data >> cy; + i++; + } + } + + for (int i = (int)arguments.size() - 1; i >= 0; --i) + { + if (!valid[i]) + { + arguments.erase(arguments.begin() + i); + } + } + + // Based on what was read in open the sequence + if (!input_image_files.empty()) + { + return OpenImageFiles(input_image_files, fx, fy, cx, cy); + } + if (!input_directory.empty()) + { + return OpenDirectory(input_directory, fx, fy, cx, cy); + } + + // If no input found return false and set a flag for it + no_input_specified = true; + + return false; +} + +// TODO proper destructors and move constructors + +bool ImageCapture::OpenImageFiles(const std::vector& image_files, float fx, float fy, float cx, float cy) +{ + + no_input_specified = false; + + latest_frame = cv::Mat(); + latest_gray_frame = cv::Mat(); + this->image_files = image_files; + + // Allow for setting the camera intrinsics, but have to be the same ones for every image + if (fx != -1 && fy != -1 && cx != -1 && cy != -1) + { + image_intrinsics_set = true; + this->fx = fx; + this->fy = fy; + this->cx = cx; + this->cy = cy; + } + + return true; + +} + +bool ImageCapture::OpenDirectory(std::string directory, float fx, float fy, float cx, float cy) +{ + INFO_STREAM("Attempting to read from directory: " << directory); + + no_input_specified = false; + + image_files.clear(); + + boost::filesystem::path image_directory(directory); + std::vector file_in_directory; + copy(boost::filesystem::directory_iterator(image_directory), boost::filesystem::directory_iterator(), back_inserter(file_in_directory)); + + // Sort the images in the directory first + sort(file_in_directory.begin(), file_in_directory.end()); + + std::vector curr_dir_files; + + for (std::vector::const_iterator file_iterator(file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator) + { + // Possible image extension .jpg and .png + if (file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".jpeg") == 0 || file_iterator->extension().string().compare(".png") == 0 || file_iterator->extension().string().compare(".bmp") == 0) + { + curr_dir_files.push_back(file_iterator->string()); + } + } + + image_files = curr_dir_files; + + if (image_files.empty()) + { + std::cout << "No images found in the directory: " << directory << std::endl; + return false; + } + + // Allow for setting the camera intrinsics, but have to be the same ones for every image + if (fx != -1 && fy != -1 && cx != -1 && cy != -1) + { + image_intrinsics_set = true; + this->fx = fx; + this->fy = fy; + this->cx = cx; + this->cy = cy; + } + + return true; + +} + +// TODO this should be shared somewhere +void convertToGrayscale(const cv::Mat& in, cv::Mat& out) +{ + if (in.channels() == 3) + { + // Make sure it's in a correct format + if (in.depth() != CV_8U) + { + if (in.depth() == CV_16U) + { + cv::Mat tmp = in / 256; + tmp.convertTo(tmp, CV_8U); + cv::cvtColor(tmp, out, CV_BGR2GRAY); + } + } + else + { + cv::cvtColor(in, out, CV_BGR2GRAY); + } + } + else if (in.channels() == 4) + { + cv::cvtColor(in, out, CV_BGRA2GRAY); + } + else + { + if (in.depth() == CV_16U) + { + cv::Mat tmp = in / 256; + out = tmp.clone(); + } + else if (in.depth() != CV_8U) + { + in.convertTo(out, CV_8U); + } + else + { + out = in.clone(); + } + } +} + +void ImageCapture::SetCameraIntrinsics(float fx, float fy, float cx, float cy) +{ + // If optical centers are not defined just use center of image + if (cx == -1) + { + this->cx = this->image_width / 2.0f; + this->cy = this->image_height / 2.0f; + } + else + { + this->cx = cx; + this->cy = cy; + } + // Use a rough guess-timate of focal length + if (fx == -1) + { + this->fx = 500.0f * (this->image_width / 640.0f); + this->fy = 500.0f * (this->image_height / 480.0f); + + this->fx = (this->fx + this->fy) / 2.0f; + this->fy = this->fx; + } + else + { + this->fx = fx; + this->fy = fy; + } +} + +cv::Mat ImageCapture::GetNextImage() +{ + frame_num++; + if (image_files.empty() || frame_num - 1 > image_files.size()) + { + // Indicate lack of success by returning an empty image + latest_frame = cv::Mat(); + } + + latest_frame = cv::imread(image_files[frame_num - 1], -1); + + if (latest_frame.empty()) + { + ERROR_STREAM("Could not open the image: " + image_files[frame_num - 1]); + } + + image_height = latest_frame.size().height; + image_width = latest_frame.size().width; + + // Reset the intrinsics for every image if they are not set globally + if (!image_intrinsics_set) + { + SetCameraIntrinsics(-1, -1, -1, -1); + } + + // Set the grayscale frame + convertToGrayscale(latest_frame, latest_gray_frame); + + this->name = boost::filesystem::path(image_files[frame_num - 1]).filename().replace_extension("").string(); + + return latest_frame; +} + +double ImageCapture::GetProgress() +{ + return (double)frame_num / (double)image_files.size(); +} + +cv::Mat_ ImageCapture::GetGrayFrame() +{ + return latest_gray_frame; +} \ No newline at end of file