diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 5943aaf..0fe9fca 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -98,8 +98,13 @@ vector get_arguments(int argc, char **argv) void get_visualization_params(bool& visualize_track, bool& visualize_align, bool& visualize_hog, vector &arguments); +// Some globals for tracking timing information for visualisation (TODO bit ugly) +double fps_tracker = -1.0; +int64 t0 = 0; +int frame_count = 0; + // Visualising the results TODO separate class -void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, int frame_count, double fx, double fy, double cx, double cy) +void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, double fx, double fy, double cx, double cy) { // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised @@ -136,20 +141,20 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f } // Work out the framerate TODO - //if (frame_count % 10 == 0) - //{ - // double t1 = cv::getTickCount(); - // fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency()); - // t0 = t1; - //} - - //// Write out the framerate on the image before displaying it - //char fpsC[255]; - //std::sprintf(fpsC, "%d", (int)fps_tracker); - //string fpsSt("FPS:"); - //fpsSt += fpsC; - //cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); + if (frame_count % 10 == 0) + { + double t1 = cv::getTickCount(); + fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency()); + t0 = t1; + } + // Write out the framerate on the image before displaying it + char fpsC[255]; + std::sprintf(fpsC, "%d", (int)fps_tracker); + string fpsSt("FPS:"); + fpsSt += fpsC; + cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); + frame_count++; } int main (int argc, char **argv) @@ -190,10 +195,11 @@ int main (int argc, char **argv) Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.fps); Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); - int frame_count = 0; - captured_image = sequence_reader.GetNextFrame(); + // For reporting progress + double reported_completion = 0; + INFO_STREAM("Starting tracking"); while (!captured_image.empty()) { @@ -224,7 +230,7 @@ int main (int argc, char **argv) // As this can be expensive only compute it if needed by output or visualization if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualize_align || visualize_hog) { - face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, time_stamp, false, !det_parameters.quiet_mode); + face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, false, !det_parameters.quiet_mode); face_analyser.GetLatestAlignedFace(sim_warped_img); if (!det_parameters.quiet_mode && visualize_align) @@ -250,7 +256,7 @@ int main (int argc, char **argv) // Drawing the visualization on the captured image if (recording_params.outputTrackedVideo() || (visualize_track && !det_parameters.quiet_mode)) { - visualise_tracking(captured_image, face_model, det_parameters, gazeDirection0, gazeDirection1, frame_count, fx, fy, cx, cy); + visualise_tracking(captured_image, face_model, det_parameters, gazeDirection0, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); } // Setting up the recorder output @@ -258,9 +264,10 @@ int main (int argc, char **argv) open_face_rec.SetObservationVisualization(captured_image); open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model)); - open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(fx, fy, cx, cy), face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success); + open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), + face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success); open_face_rec.SetObservationPose(pose_estimate); - open_face_rec.SetObservationTimestamp(time_stamp); + open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp); open_face_rec.WriteObservation(); // Visualize the image if desired @@ -290,20 +297,18 @@ int main (int argc, char **argv) } } - if(total_frames != -1) + + if(sequence_reader.GetProgress() >= reported_completion / 10.0) { - if((double)frame_count/(double)total_frames >= reported_completion / 10.0) - { - cout << reported_completion * 10 << "% "; - reported_completion = reported_completion + 1; - } + cout << reported_completion * 10 << "% "; + reported_completion = reported_completion + 1; } } open_face_rec.Close(); - if (output_files.size() > 0 && recording_params.outputAUs()) + if (recording_params.outputAUs()) { cout << "Postprocessing the Action Unit predictions" << endl; face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile()); diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h index 8ab8b0b..a8411f4 100644 --- a/lib/local/Utilities/include/SequenceCapture.h +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -79,6 +79,7 @@ namespace Utilities // Getting the most recent grayscale frame (need to call GetNextFrame first) cv::Mat_ GetGrayFrame(); + // Parameters describing the sequence and it's progress double GetProgress(); bool IsOpened(); @@ -90,6 +91,8 @@ namespace Utilities double fps; + double time_stamp; + // Name of the video file, image directory, or the webcam std::string name; @@ -116,6 +119,8 @@ namespace Utilities bool img_grabbed; + // If using a webcam, helps to keep track of time + int64 start_time; }; } #endif \ No newline at end of file diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 597c3fb..aab0662 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -157,7 +157,7 @@ void RecorderOpenFace::WriteObservation() { // As we are writing out the header, work out some things like number of landmarks, names of AUs etc. int num_face_landmarks = landmarks_2D.rows / 2; - int num_eye_landmarks = eye_landmarks.size(); + int num_eye_landmarks = (int)eye_landmarks.size(); int num_model_modes = pdm_params_local.rows; std::vector au_names_class; diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index e68f510..2c55452 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -142,7 +142,7 @@ bool SequenceCapture::Open(std::vector arguments) } } - for (int i = arguments.size() - 1; i >= 0; --i) + for (size_t i = arguments.size() - 1; i >= 0; --i) { if (!valid[i]) { @@ -190,8 +190,8 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, vid_length = 0; frame_num = 0; - this->frame_width = capture.get(CV_CAP_PROP_FRAME_WIDTH); - this->frame_height = capture.get(CV_CAP_PROP_FRAME_HEIGHT); + this->frame_width = (int)capture.get(CV_CAP_PROP_FRAME_WIDTH); + this->frame_height = (int)capture.get(CV_CAP_PROP_FRAME_HEIGHT); if (!capture.isOpened()) { @@ -222,14 +222,16 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, // Use a rough guess-timate of focal length if (fx == -1) { - fx = 500 * (frame_width / 640.0); - fy = 500 * (frame_height / 480.0); + fx = (float)(500.0 * (frame_width / 640.0)); + fy = (float)(500.0 * (frame_height / 480.0)); - fx = (fx + fy) / 2.0; + fx = (fx + fy) / 2.0f; fy = fx; } this->name = "webcam"; // TODO number + + start_time = cv::getTickCount(); return true; @@ -258,10 +260,10 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, is_webcam = false; is_image_seq = false; - this->frame_width = capture.get(CV_CAP_PROP_FRAME_WIDTH); - this->frame_height = capture.get(CV_CAP_PROP_FRAME_HEIGHT); + this->frame_width = (int)capture.get(CV_CAP_PROP_FRAME_WIDTH); + this->frame_height = (int)capture.get(CV_CAP_PROP_FRAME_HEIGHT); - vid_length = capture.get(CV_CAP_PROP_FRAME_COUNT); + vid_length = (int)capture.get(CV_CAP_PROP_FRAME_COUNT); frame_num = 0; if (capture.isOpened()) @@ -279,14 +281,14 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, // Use a rough guess-timate of focal length if (fx == -1) { - fx = 500 * (frame_width / 640.0); - fy = 500 * (frame_height / 480.0); + fx = 500.0f * (frame_width / 640.0f); + fy = 500.0f * (frame_height / 480.0f); - fx = (fx + fy) / 2.0; + fx = (fx + fy) / 2.0f; fy = fx; } - this->name = boost::filesystem::path(video_file).filename.replace_extension("").string(); + this->name = boost::filesystem::path(video_file).filename().replace_extension("").string(); return true; @@ -338,17 +340,17 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f // Use a rough guess-timate of focal length if (fx == -1) { - fx = 500 * (frame_width / 640.0); - fy = 500 * (frame_height / 480.0); + fx = 500.0f * (frame_width / 640.0f); + fy = 500.0f * (frame_height / 480.0f); - fx = (fx + fy) / 2.0; + fx = (fx + fy) / 2.0f; fy = fx; } // No fps as we have a sequence this->fps = 0; - this->name = boost::filesystem::path(directory).filename.string(); + this->name = boost::filesystem::path(directory).filename().string(); return true; @@ -400,7 +402,7 @@ cv::Mat SequenceCapture::GetNextFrame() { frame_num++; - if (is_webcam && !is_image_seq) + if (is_webcam || !is_image_seq) { bool success = capture.read(latest_frame); @@ -410,6 +412,17 @@ cv::Mat SequenceCapture::GetNextFrame() // Indicate lack of success by returning an empty image latest_frame = cv::Mat(); } + + // Recording the timestamp + if (!is_webcam) + { + time_stamp = frame_num * (1.0 / fps); + } + else + { + time_stamp = (cv::getTickCount() - start_time) / cv::getTickFrequency(); + } + } else if (is_image_seq) { @@ -420,6 +433,7 @@ cv::Mat SequenceCapture::GetNextFrame() } latest_frame = cv::imread(image_files[frame_num-1], -1); + time_stamp = 0; } // Set the grayscale frame