From e629ae50be36f2fd60bd87f996a1528b84f3442b Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Tue, 12 Dec 2017 08:55:23 +0000 Subject: [PATCH] A number of fixes to the recorder: - camera parameters in the meta file - fixing bug with video output - Relative and full path input in the meta file --- exe/FaceLandmarkImg/FaceLandmarkImg.cpp | 3 +- exe/FeatureExtraction/FeatureExtraction.cpp | 3 +- .../include/RecorderOpenFaceParameters.h | 10 ++- lib/local/Utilities/include/SequenceCapture.h | 3 - lib/local/Utilities/src/RecorderOpenFace.cpp | 75 +++++++++++++++---- .../src/RecorderOpenFaceParameters.cpp | 7 +- lib/local/Utilities/src/SequenceCapture.cpp | 18 +---- 7 files changed, 82 insertions(+), 37 deletions(-) diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index e03e0a5..71c0ab2 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -186,7 +186,8 @@ int main (int argc, char **argv) while (!captured_image.empty()) { - Utilities::RecorderOpenFaceParameters recording_params(arguments, false); + Utilities::RecorderOpenFaceParameters recording_params(arguments, false, + image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy); Utilities::RecorderOpenFace open_face_rec(image_reader.name, recording_params, arguments); visualizer.SetImage(captured_image, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy); diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 97e28c0..dad67d8 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -140,7 +140,8 @@ int main (int argc, char **argv) cv::Mat captured_image; - Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.fps); + Utilities::RecorderOpenFaceParameters recording_params(arguments, true, + sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps); Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); if (recording_params.outputGaze() && !face_model.eye_model) diff --git a/lib/local/Utilities/include/RecorderOpenFaceParameters.h b/lib/local/Utilities/include/RecorderOpenFaceParameters.h index 66df54c..1dbb462 100644 --- a/lib/local/Utilities/include/RecorderOpenFaceParameters.h +++ b/lib/local/Utilities/include/RecorderOpenFaceParameters.h @@ -53,7 +53,7 @@ namespace Utilities public: // Constructors - RecorderOpenFaceParameters(std::vector &arguments, bool sequence, double fps_vid_out = 30); + RecorderOpenFaceParameters(std::vector &arguments, bool sequence, float fx = -1, float fy = -1, float cx = -1, float cy = -1, double fps_vid_out = 30); bool isSequence() const { return is_sequence; } bool output2DLandmarks() const { return output_2D_landmarks; } @@ -68,6 +68,11 @@ namespace Utilities std::string outputCodec() const { return output_codec; } double outputFps() const { return fps_vid_out; } + float getFx() const { return fx; } + float getFy() const { return fy; } + float getCx() const { return cx; } + float getCy() const { return cy; } + private: // If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face @@ -88,6 +93,9 @@ namespace Utilities std::string output_codec; double fps_vid_out; + // Camera parameters for recording in the meta file; + float fx, fy, cx, cy; + }; } diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h index 874a254..8995b7b 100644 --- a/lib/local/Utilities/include/SequenceCapture.h +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -95,9 +95,6 @@ namespace Utilities double time_stamp; - std::string input_name_relative; - std::string input_name_full; - // Name of the video file, image directory, or the webcam std::string name; diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index 86af070..7ff5307 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -76,7 +76,14 @@ RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFa { // From the filename, strip out the name without directory and extension - filename = in_filename; + if (boost::filesystem::is_directory(in_filename)) + { + filename = boost::filesystem::canonical(boost::filesystem::path(in_filename)).filename().string(); + } + else + { + filename = boost::filesystem::path(in_filename).filename().replace_extension("").string(); + } // Consuming the input arguments bool* valid = new bool[arguments.size()]; @@ -136,18 +143,37 @@ RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFa exit(1); } - // Populate the metadata file - metadata_file << "Input:" << in_filename << endl; + // Populate relative and full path names in the meta file, unless it is a webcam + if(in_filename.compare("webcam") != 0) + { + string input_filename_relative = in_filename; + string input_filename_full = in_filename; + if (!boost::filesystem::path(input_filename_full).is_absolute()) + { + input_filename_full = boost::filesystem::canonical(input_filename_relative).string(); + } + metadata_file << "Input:" << input_filename_relative << endl; + metadata_file << "Input full path:" << input_filename_full << endl; + } + else + { + // Populate the metadata file + metadata_file << "Input:" << in_filename << endl; + } + + metadata_file << "Camera parameters:" << parameters.getFx() << "," << parameters.getFy() << parameters.getCx() << "," << parameters.getCy() << endl; // Create the required individual recorders, CSV, HOG, aligned, video - csv_filename = (path(record_root) / path(filename).concat(".csv")).string(); + csv_filename = path(filename).concat(".csv").string(); // Consruct HOG recorder here if(params.outputHOG()) { - std::string hog_filename = (path(record_root) / path(filename).replace_extension(".hog")).string(); + // Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory + std::string hog_filename = path(filename).concat(".hog").string(); + metadata_file << "Output HOG:" << hog_filename << endl; + hog_filename = (path(record_root) / hog_filename).string(); hog_recorder.Open(hog_filename); - metadata_file << "Output HOG:" << csv_filename << endl; } // saving the videos @@ -155,25 +181,28 @@ RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFa { if(parameters.isSequence()) { - this->media_filename = (path(record_root) / path(filename).replace_extension(".avi")).string(); + // Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory + this->media_filename = path(filename).concat(".avi").string(); metadata_file << "Output video:" << this->media_filename << endl; + this->media_filename = (path(record_root) / this->media_filename).string(); } else { - this->media_filename = (path(record_root) / path(filename).replace_extension(".jpg")).string(); + this->media_filename = path(filename).concat(".jpg").string(); metadata_file << "Output image:" << this->media_filename << endl; + this->media_filename = (path(record_root) / this->media_filename).string(); } } // Prepare image recording if (params.outputAlignedFaces()) { - aligned_output_directory = (path(record_root) / path(filename + "_aligned")).string(); - CreateDirectory(aligned_output_directory); + aligned_output_directory = path(filename + "_aligned").string(); metadata_file << "Output aligned directory:" << this->aligned_output_directory << endl; + this->aligned_output_directory = (path(record_root) / this->aligned_output_directory).string(); + CreateDirectory(aligned_output_directory); } - - + observation_count = 0; } @@ -188,17 +217,23 @@ void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track) if (params.outputTracked()) { // Initialize the video writer if it has not been opened yet - if(params.isSequence()) + if(params.isSequence() && !video_writer.isOpened()) { std::string output_codec = params.outputCodec(); try { video_writer.open(media_filename, CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), params.outputFps(), vis_track.size(), true); + + if (!video_writer.isOpened()) + { + WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN."); + } } catch (cv::Exception e) { WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); } + } vis_to_out = vis_track; @@ -240,7 +275,12 @@ void RecorderOpenFace::WriteObservation() params.outputAUs(), params.outputGaze(), num_face_landmarks, num_model_modes, num_eye_landmarks, au_names_class, au_names_reg); metadata_file << "Output csv:" << csv_filename << endl; - + metadata_file << "Gaze: " << params.outputGaze() << endl; + metadata_file << "AUs: " << params.outputAUs() << endl; + metadata_file << "Landmarks 2D: " << params.output2DLandmarks() << endl; + metadata_file << "Landmarks 3D: " << params.output3DLandmarks() << endl; + metadata_file << "Pose: " << params.outputPose() << endl; + metadata_file << "Shape parameters: " << params.outputPDMParams() << endl; } this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success, @@ -284,9 +324,12 @@ void RecorderOpenFace::WriteObservation() WARN_STREAM("Output tracked video frame is not set"); } - if(video_writer.isOpened()) + if(params.isSequence() ) { - video_writer.write(vis_to_out); + if(video_writer.isOpened()) + { + video_writer.write(vis_to_out); + } } else { diff --git a/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp index 52fc819..15fff7f 100644 --- a/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp +++ b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp @@ -37,13 +37,18 @@ using namespace std; using namespace Utilities; -RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector &arguments, bool sequence, double fps_vid_out) +RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector &arguments, bool sequence, float fx, float fy, float cx, float cy, double fps_vid_out) { string separator = string(1, boost::filesystem::path::preferred_separator); this->is_sequence = sequence; + this->fx = fx; + this->fy = fy; + this->cx = cx; + this->cy = cy; + if(fps_vid_out > 0) { this->fps_vid_out = fps_vid_out; diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index cf74a81..65edd40 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -156,28 +156,18 @@ bool SequenceCapture::Open(std::vector& arguments) no_input_specified = !file_found; - - // Based on what was read in open the sequence TODO + // Based on what was read in open the sequence if (device != -1) { - input_name_relative = "webcam"; - input_name_full = "webcam"; + // TODO allow to specify webcam resolution return OpenWebcam(device, 640, 480, fx, fy, cx, cy); } if (!input_video_file.empty()) { - input_name_relative = input_video_file; - if (boost::filesystem::path(input_name_relative).is_absolute()) - { - input_name_full = input_name_relative; - } - - boost::filesystem::path(input_name_full).is_absolute(); return OpenVideoFile(input_video_file, fx, fy, cx, cy); } if (!input_sequence_directory.empty()) { - input_name_relative = input_sequence_directory; return OpenImageSequence(input_sequence_directory, fx, fy, cx, cy); } @@ -296,7 +286,7 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, SetCameraIntrinsics(fx, fy, cx, cy); - this->name = boost::filesystem::path(video_file).filename().replace_extension("").string(); + this->name = video_file; return true; @@ -346,7 +336,7 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f // No fps as we have a sequence this->fps = 0; - this->name = boost::filesystem::path(directory).filename().string(); + this->name = directory; is_webcam = false; is_image_seq = true;