From 6c14aeab895cc73070d34c4aa6d3afd17a33397b Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Thu, 9 Nov 2017 18:25:49 +0000 Subject: [PATCH] Big cleanup for SequenceCapture --- exe/FeatureExtraction/FeatureExtraction.cpp | 186 +++--------------- lib/local/Utilities/include/SequenceCapture.h | 6 +- lib/local/Utilities/src/RecorderOpenFace.cpp | 4 - lib/local/Utilities/src/SequenceCapture.cpp | 25 ++- 4 files changed, 54 insertions(+), 167 deletions(-) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index a90132b..358ca3e 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -98,7 +98,7 @@ vector get_arguments(int argc, char **argv) void get_visualization_params(bool& visualize_track, bool& visualize_align, bool& visualize_hog, vector &arguments); -// Visualising the results +// Visualising the results TODO separate class void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, int frame_count, double fx, double fy, double cx, double cy) { @@ -135,20 +135,20 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f } } - // Work out the framerate - if (frame_count % 10 == 0) - { - double t1 = cv::getTickCount(); - fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency()); - t0 = t1; - } + // Work out the framerate TODO + //if (frame_count % 10 == 0) + //{ + // double t1 = cv::getTickCount(); + // fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency()); + // t0 = t1; + //} - // Write out the framerate on the image before displaying it - char fpsC[255]; - std::sprintf(fpsC, "%d", (int)fps_tracker); - string fpsSt("FPS:"); - fpsSt += fpsC; - cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); + //// Write out the framerate on the image before displaying it + //char fpsC[255]; + //std::sprintf(fpsC, "%d", (int)fps_tracker); + //string fpsSt("FPS:"); + //fpsSt += fpsC; + //cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); } @@ -174,72 +174,35 @@ int main (int argc, char **argv) FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); - Utilities::SequenceCapture sequence_reader(); + Utilities::SequenceCapture sequence_reader; while (true) // this is not a for loop as we might also be reading from a webcam { - // INFO_STREAM("Attempting to read from file: " << current_file); TODO add reading info stuff - // INFO_STREAM("FPS of the video file cannot be determined, assuming 30"); - // INFO_STREAM("Device or file opened"); - - sequence_reader.Open(arguments); - if (!sequence_reader.IsOpen()) + // The sequence reader chooses what to open based on command line arguments provided + if(!sequence_reader.Open(arguments)) break; - string current_file; + INFO_STREAM("Device or file opened"); cv::Mat captured_image; - Utilities::RecorderOpenFaceParameters recording_params(arguments, true, fps_vid_in); - Utilities::RecorderOpenFace open_face_rec(output_files[f_n], input_files[f_n], recording_params); + Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.fps); + Utilities::RecorderOpenFace open_face_rec(output_files[f_n], sequence_reader.name, recording_params); int frame_count = 0; - // Use for timestamping if using a webcam - int64 t_initial = cv::getTickCount(); - - // Timestamp in seconds of current processing - double time_stamp = 0; + captured_image = sequence_reader.GetNextFrame(); INFO_STREAM("Starting tracking"); while (!captured_image.empty()) { - // Grab the timestamp first (TODO timestamp should be grabbed from sequence) - if (video_input) - { - time_stamp = (double)frame_count * (1.0 / fps_vid_in); - } - else - { - // if loading images assume 30fps - time_stamp = (double)frame_count * (1.0 / 30.0); - } - - // Reading the images, TODO grayscale should be grabbed another way - cv::Mat_ grayscale_image; - - if (captured_image.channels() == 3) - { - cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); - } - else - { - grayscale_image = captured_image.clone(); - } + // Converting to grayscale + cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); // The actual facial landmark detection / tracking - bool detection_success; - - if (video_input || images_as_video) - { - detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters); - } - else - { - detection_success = LandmarkDetector::DetectLandmarksInImage(grayscale_image, face_model, det_parameters); - } + bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters); // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); @@ -308,24 +271,8 @@ int main (int argc, char **argv) } // Grabbing the next frame (todo this should be part of capture) - if(video_input) - { - video_capture >> captured_image; - } - else - { - curr_img++; - if(curr_img < (int)input_image_files[f_n].size()) - { - string curr_img_file = input_image_files[f_n][curr_img]; - captured_image = cv::imread(curr_img_file, -1); - } - else - { - captured_image = cv::Mat(); - } - } - + captured_image = sequence_reader.GetNextFrame(); + if (!det_parameters.quiet_mode) { // detect key presses @@ -343,9 +290,6 @@ int main (int argc, char **argv) } } - // Update the frame count - frame_count++; - if(total_frames != -1) { if((double)frame_count/(double)total_frames >= reported_completion / 10.0) @@ -369,19 +313,6 @@ int main (int argc, char **argv) face_analyser.Reset(); face_model.Reset(); - frame_count = 0; - curr_img = -1; - - if (total_frames != -1) - { - cout << endl; - } - - // break out of the loop if done with all the files (or using a webcam) - if((video_input && f_n == input_files.size() -1) || (!video_input && f_n == input_image_files.size() - 1)) - { - done = true; - } } return 0; @@ -438,68 +369,3 @@ void get_visualization_params(bool& visualize_track, bool& visualize_align, bool } -// Can process images via directories creating a separate output file per directory -void get_image_input_output_params_feats(vector > &input_image_files, bool& as_video, vector &arguments) -{ - bool* valid = new bool[arguments.size()]; - - for (size_t i = 0; i < arguments.size(); ++i) - { - valid[i] = true; - if (arguments[i].compare("-fdir") == 0) - { - - // parse the -fdir directory by reading in all of the .png and .jpg files in it - path image_directory(arguments[i + 1]); - - try - { - // does the file exist and is it a directory - if (exists(image_directory) && is_directory(image_directory)) - { - - vector file_in_directory; - copy(directory_iterator(image_directory), directory_iterator(), back_inserter(file_in_directory)); - - // Sort the images in the directory first - sort(file_in_directory.begin(), file_in_directory.end()); - - vector curr_dir_files; - - for (vector::const_iterator file_iterator(file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator) - { - // Possible image extension .jpg and .png - if (file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".png") == 0) - { - curr_dir_files.push_back(file_iterator->string()); - } - } - - input_image_files.push_back(curr_dir_files); - } - } - catch (const filesystem_error& ex) - { - cout << ex.what() << '\n'; - } - - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-asvid") == 0) - { - as_video = true; - } - } - - // Clear up the argument list - for (int i = arguments.size() - 1; i >= 0; --i) - { - if (!valid[i]) - { - arguments.erase(arguments.begin() + i); - } - } - -} \ No newline at end of file diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h index f639fed..8ab8b0b 100644 --- a/lib/local/Utilities/include/SequenceCapture.h +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -88,6 +88,11 @@ namespace Utilities float fx, fy, cx, cy; + double fps; + + // Name of the video file, image directory, or the webcam + std::string name; + private: // Used for capturing webcam and video @@ -97,7 +102,6 @@ namespace Utilities cv::Mat latest_frame; cv::Mat latest_gray_frame; - double fps; // Keeping track if we are opening a video, webcam or image sequence bool is_webcam; diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 1d5a791..1eeb4ae 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -109,14 +109,11 @@ RecorderOpenFace::RecorderOpenFace(const std::string out_directory, const std::s } -// TODO move to actual write void RecorderOpenFace::SetObservationFaceAlign(const cv::Mat& aligned_face) { this->aligned_face = aligned_face; - } - void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) { if (params.outputTrackedVideo()) @@ -142,7 +139,6 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) } - void RecorderOpenFace::WriteObservation() { observation_count++; diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index d002b69..e68f510 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -45,6 +45,15 @@ using namespace Utilities; +#define INFO_STREAM( stream ) \ +std::cout << stream << std::endl + +#define WARN_STREAM( stream ) \ +std::cout << "Warning: " << stream << std::endl + +#define ERROR_STREAM( stream ) \ +std::cout << "Error: " << stream << std::endl + bool SequenceCapture::Open(std::vector arguments) { @@ -154,11 +163,13 @@ bool SequenceCapture::Open(std::vector arguments) { return OpenImageSequence(input_sequence_directory, fx, fy, cx, cy); } - + // If none opened, return false + return false; } bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, float fx, float fy, float cx, float cy) { + INFO_STREAM("Attempting to read from webcam: " << device); if (device < 0) { @@ -218,6 +229,8 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, fy = fx; } + this->name = "webcam"; // TODO number + return true; } @@ -226,6 +239,8 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, float cx, float cy) { + INFO_STREAM("Attempting to read from file: " << video_file); + latest_frame = cv::Mat(); latest_gray_frame = cv::Mat(); @@ -236,7 +251,7 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, // Check if fps is nan or less than 0 if (fps != fps || fps <= 0) { - INFO_STREAM("FPS of the video file cannot be determined, assuming 30"); + WARN_STREAM("FPS of the video file cannot be determined, assuming 30"); fps = 30; } @@ -271,12 +286,16 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, fy = fx; } + this->name = boost::filesystem::path(video_file).filename.replace_extension("").string(); + return true; } bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float fy, float cx, float cy) { + INFO_STREAM("Attempting to read from directory: " << directory); + image_files.clear(); boost::filesystem::path image_directory(directory); @@ -329,6 +348,8 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f // No fps as we have a sequence this->fps = 0; + this->name = boost::filesystem::path(directory).filename.string(); + return true; }