From fa48372f4ceb99ca3f20a637d2222f933a57f8a2 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Mon, 13 Nov 2017 09:07:52 +0000 Subject: [PATCH] Adding a dedicated FPS tracker. --- exe/FeatureExtraction/FeatureExtraction.cpp | 37 +++++-------------- .../Utilities/include/RecorderOpenFace.h | 2 +- .../Utilities/include/VisualizationUtils.h | 20 +++++++++- lib/local/Utilities/include/Visualizer.h | 2 + lib/local/Utilities/src/RecorderOpenFace.cpp | 11 +++++- lib/local/Utilities/src/SequenceCapture.cpp | 12 +++--- .../Utilities/src/VisualizationUtils.cpp | 33 +++++++++++++++++ lib/local/Utilities/src/Visualizer.cpp | 10 +++++ 8 files changed, 89 insertions(+), 38 deletions(-) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 00db2db..bef128d 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -60,6 +60,7 @@ #include #include #include +#include #ifndef CONFIG_DIR #define CONFIG_DIR "~" @@ -97,34 +98,6 @@ vector get_arguments(int argc, char **argv) return arguments; } -// Some globals for tracking timing information for visualisation (TODO bit ugly) -double fps_tracker = -1.0; -int64 t0 = 0; -int frame_count = 0; - -// Visualising the results TODO separate class -void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, double fx, double fy, double cx, double cy) -{ - - - // Work out the framerate TODO - if (frame_count % 10 == 0) - { - double t1 = cv::getTickCount(); - fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency()); - t0 = t1; - } - - // Write out the framerate on the image before displaying it - char fpsC[255]; - std::sprintf(fpsC, "%d", (int)fps_tracker); - string fpsSt("FPS:"); - fpsSt += fpsC; - cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); - frame_count++; - -} - int main (int argc, char **argv) { @@ -146,6 +119,10 @@ int main (int argc, char **argv) // A utility for visualizing the results Utilities::Visualizer visualizer(arguments); + // Tracking FPS for visualization + Utilities::FpsTracker fps_tracker; + fps_tracker.AddFrame(); + while (true) // this is not a for loop as we might also be reading from a webcam { @@ -200,6 +177,9 @@ int main (int argc, char **argv) // Work out the pose of the head from the tracked model cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + // Keeping track of FPS + fps_tracker.AddFrame(); + // Displaying the tracking visualizations visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); visualizer.SetObservationFaceAlign(sim_warped_img); @@ -207,6 +187,7 @@ int main (int argc, char **argv) visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, detection_success); visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); + visualizer.SetFps(fps_tracker.GetFPS()); visualizer.ShowObservation(); // Setting up the recorder output diff --git a/lib/local/Utilities/include/RecorderOpenFace.h b/lib/local/Utilities/include/RecorderOpenFace.h index fe2179b..ab66927 100644 --- a/lib/local/Utilities/include/RecorderOpenFace.h +++ b/lib/local/Utilities/include/RecorderOpenFace.h @@ -57,7 +57,7 @@ namespace Utilities public: // The constructor for the recorder, need to specify if we are recording a sequence or not - RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::vector arguments); + RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::vector& arguments); ~RecorderOpenFace(); diff --git a/lib/local/Utilities/include/VisualizationUtils.h b/lib/local/Utilities/include/VisualizationUtils.h index 8c4d452..518610f 100644 --- a/lib/local/Utilities/include/VisualizationUtils.h +++ b/lib/local/Utilities/include/VisualizationUtils.h @@ -37,13 +37,13 @@ #include #include +#include namespace Utilities { // TODO draw AU results - // Drawing a bounding box around the face in an image void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy); void DrawBox(const std::vector>& lines, cv::Mat image, cv::Scalar color, int thickness); @@ -53,6 +53,24 @@ namespace Utilities void Visualise_FHOG(const cv::Mat_& descriptor, int num_rows, int num_cols, cv::Mat& visualisation); + class FpsTracker + { + public: + + double history_length; + + void AddFrame(); + + double GetFPS(); + + FpsTracker(); + + private: + std::queue frame_times; + + void DiscardOldFrames(); + + }; } #endif \ No newline at end of file diff --git a/lib/local/Utilities/include/Visualizer.h b/lib/local/Utilities/include/Visualizer.h index b6e2c7f..9344d87 100644 --- a/lib/local/Utilities/include/Visualizer.h +++ b/lib/local/Utilities/include/Visualizer.h @@ -80,6 +80,8 @@ namespace Utilities // HOG feature related observations void SetObservationHOG(const cv::Mat_& hog_descriptor, int num_cols, int num_rows); + void SetFps(double fps); + void ShowObservation(); cv::Mat GetVisImage(); diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 8d6172c..965424f 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -71,7 +71,7 @@ void CreateDirectory(std::string output_path) } -RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::vector arguments):video_writer(), params(parameters) +RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::vector& arguments):video_writer(), params(parameters) { // From the filename, strip out the name without directory and extension @@ -150,7 +150,14 @@ RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFa if (params.outputTrackedVideo()) { this->video_filename = (path(record_root) / path(filename).replace_extension(".avi")).string(); - metadata_file << "Output video:" << this->video_filename << endl; + if(parameters.isSequence()) + { + metadata_file << "Output video:" << this->video_filename << endl; + } + else + { + metadata_file << "Output image:" << this->video_filename << endl; + } } // Prepare image recording diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index c59f809..cff1106 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -246,6 +246,12 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, capture.open(video_file); + if (!capture.isOpened()) + { + std::cout << "Failed to open the video file at location: " << video_file << std::endl; + return false; + } + this->fps = capture.get(CV_CAP_PROP_FPS); // Check if fps is nan or less than 0 @@ -263,12 +269,6 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, vid_length = (int)capture.get(CV_CAP_PROP_FRAME_COUNT); - if (!capture.isOpened()) - { - std::cout << "Failed to open the video file at location: " << video_file << std::endl; - return false; - } - SetCameraIntrinsics(fx, fy, cx, cy); this->name = boost::filesystem::path(video_file).filename().replace_extension("").string(); diff --git a/lib/local/Utilities/src/VisualizationUtils.cpp b/lib/local/Utilities/src/VisualizationUtils.cpp index 090ab87..8b9f5bc 100644 --- a/lib/local/Utilities/src/VisualizationUtils.cpp +++ b/lib/local/Utilities/src/VisualizationUtils.cpp @@ -44,6 +44,39 @@ namespace Utilities { + FpsTracker::FpsTracker() + { + // Keep two seconds of history + history_length = 2; + } + + void FpsTracker::AddFrame() + { + double current_time = cv::getTickCount() / cv::getTickFrequency(); + frame_times.push(current_time); + DiscardOldFrames(); + } + + double FpsTracker::GetFPS() + { + DiscardOldFrames(); + + if (frame_times.size() == 0) + return 0; + + double current_time = cv::getTickCount() / cv::getTickFrequency(); + + return ((double)frame_times.size()) / (current_time - frame_times.front()); + } + + void FpsTracker::DiscardOldFrames() + { + double current_time = cv::getTickCount() / cv::getTickFrequency(); + // Remove old history + while (frame_times.size() > 0 && (current_time - frame_times.front()) > history_length) + frame_times.pop(); + } + void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy) { auto edge_lines = CalculateBox(pose, fx, fy, cx, cy); diff --git a/lib/local/Utilities/src/Visualizer.cpp b/lib/local/Utilities/src/Visualizer.cpp index 5c8dc9b..1a2fd6f 100644 --- a/lib/local/Utilities/src/Visualizer.cpp +++ b/lib/local/Utilities/src/Visualizer.cpp @@ -232,6 +232,16 @@ void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv } } +void Visualizer::SetFps(double fps) +{ + // Write out the framerate on the image before displaying it + char fpsC[255]; + std::sprintf(fpsC, "%d", (int)fps); + std::string fpsSt("FPS:"); + fpsSt += fpsC; + cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); +} + void Visualizer::ShowObservation() { if (vis_track)