From deec0528cb36cae213f485c620a71950a0516dee Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Tue, 16 Jan 2018 08:58:51 +0000 Subject: [PATCH] Working on the recorder interop --- lib/local/CppInerop/CppInerop.vcxproj | 1 + lib/local/CppInerop/CppInerop.vcxproj.filters | 3 + lib/local/CppInerop/RecorderInterop.h | 50 +++++- .../Utilities/include/RecorderOpenFace.h | 3 + .../include/RecorderOpenFaceParameters.h | 3 + lib/local/Utilities/src/RecorderOpenFace.cpp | 164 ++++++++++-------- .../src/RecorderOpenFaceParameters.cpp | 34 +++- 7 files changed, 184 insertions(+), 74 deletions(-) diff --git a/lib/local/CppInerop/CppInerop.vcxproj b/lib/local/CppInerop/CppInerop.vcxproj index 576a2ef..3a8b7d9 100644 --- a/lib/local/CppInerop/CppInerop.vcxproj +++ b/lib/local/CppInerop/CppInerop.vcxproj @@ -185,6 +185,7 @@ + diff --git a/lib/local/CppInerop/CppInerop.vcxproj.filters b/lib/local/CppInerop/CppInerop.vcxproj.filters index 993d109..8860193 100644 --- a/lib/local/CppInerop/CppInerop.vcxproj.filters +++ b/lib/local/CppInerop/CppInerop.vcxproj.filters @@ -44,5 +44,8 @@ Header Files + + Header Files + \ No newline at end of file diff --git a/lib/local/CppInerop/RecorderInterop.h b/lib/local/CppInerop/RecorderInterop.h index 3dec34e..60a7f02 100644 --- a/lib/local/CppInerop/RecorderInterop.h +++ b/lib/local/CppInerop/RecorderInterop.h @@ -1,6 +1,5 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, -// all rights reserved. +// Copyright (C) 2017, Tadas Baltrusaitis. // // ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY // @@ -44,7 +43,49 @@ #pragma managed -namespace Utilities { +namespace UtilitiesOF { + + public ref class RecorderOpenFaceParameters + { + + private: + + Utilities::RecorderOpenFaceParameters *m_params; + + public: + RecorderOpenFaceParameters(bool sequence, bool is_from_webcam, bool output_2D_landmarks, bool output_3D_landmarks, + bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked, + bool output_aligned_faces, float fx, float fy, float cx, float cy, double fps_vid_out) + { + + m_params = new Utilities::RecorderOpenFaceParameters(sequence, is_from_webcam, + output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, + output_gaze, output_hog, output_tracked, output_aligned_faces, fx, fy, cx, cy, fps_vid_out); + + } + + Utilities::RecorderOpenFaceParameters * GetParams() + { + return m_params; + } + + !RecorderOpenFaceParameters() + { + // Automatically closes capture object before freeing memory. + if (m_params != nullptr) + { + delete m_params; + } + + } + + // Destructor. Called on explicit Dispose() only. + ~RecorderOpenFaceParameters() + { + this->!RecorderOpenFaceParameters(); + } + + }; public ref class RecorderOpenFace { @@ -56,8 +97,9 @@ namespace Utilities { public: // Can provide a directory, or a list of files - RecorderOpenFace() + RecorderOpenFace(const std::string in_filename, UtilitiesOF::RecorderOpenFaceParameters^ parameters, std::string output_directory, std::string output_name) { + m_recorder = new Utilities::RecorderOpenFace(in_filename, parameters->GetParams(), output_directory, output_name); } diff --git a/lib/local/Utilities/include/RecorderOpenFace.h b/lib/local/Utilities/include/RecorderOpenFace.h index 25bdd13..8725ef8 100644 --- a/lib/local/Utilities/include/RecorderOpenFace.h +++ b/lib/local/Utilities/include/RecorderOpenFace.h @@ -58,6 +58,7 @@ namespace Utilities // The constructor for the recorder, need to specify if we are recording a sequence or not, in_filename should be just the name and not contain extensions RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::vector& arguments); + RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::string output_directory, std::string output_name); ~RecorderOpenFace(); @@ -104,6 +105,8 @@ namespace Utilities RecorderOpenFace(const RecorderOpenFace&& other); RecorderOpenFace(const RecorderOpenFace& other); + void PrepareRecording(std::string in_filename); + // Keeping track of what to output and how to output it const RecorderOpenFaceParameters params; diff --git a/lib/local/Utilities/include/RecorderOpenFaceParameters.h b/lib/local/Utilities/include/RecorderOpenFaceParameters.h index d2cbf16..2f791e7 100644 --- a/lib/local/Utilities/include/RecorderOpenFaceParameters.h +++ b/lib/local/Utilities/include/RecorderOpenFaceParameters.h @@ -54,6 +54,9 @@ namespace Utilities // Constructors RecorderOpenFaceParameters(std::vector &arguments, bool sequence, bool is_from_webcam, float fx = -1, float fy = -1, float cx = -1, float cy = -1, double fps_vid_out = 30); + RecorderOpenFaceParameters(bool sequence, bool is_from_webcam, bool output_2D_landmarks, bool output_3D_landmarks, + bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked, + bool output_aligned_faces, float fx = -1, float fy = -1, float cx = -1, float cy = -1, double fps_vid_out = 30); bool isSequence() const { return is_sequence; } bool isFromWebcam() const { return is_from_webcam; } diff --git a/lib/local/Utilities/src/RecorderOpenFace.cpp b/lib/local/Utilities/src/RecorderOpenFace.cpp index d8ff222..24ef597 100644 --- a/lib/local/Utilities/src/RecorderOpenFace.cpp +++ b/lib/local/Utilities/src/RecorderOpenFace.cpp @@ -71,6 +71,87 @@ void CreateDirectory(std::string output_path) } } +void RecorderOpenFace::PrepareRecording(std::string in_filename) +{ + // Construct the directories required for the output + CreateDirectory(record_root); + + // Create the filename for the general output file that contains all of the meta information about the recording + path of_det_name(filename); + of_det_name = path(record_root) / path(filename + "_of_details.txt"); + + // Write in the of file what we are outputing what is the input etc. + metadata_file.open(of_det_name.string(), std::ios_base::out); + if (!metadata_file.is_open()) + { + cout << "ERROR: could not open the output file:" << of_det_name << ", either the path of the output directory is wrong or you do not have the permissions to write to it" << endl; + exit(1); + } + + // Populate relative and full path names in the meta file, unless it is a webcam + if (!params.isFromWebcam()) + { + string input_filename_relative = in_filename; + string input_filename_full = in_filename; + if (!boost::filesystem::path(input_filename_full).is_absolute()) + { + input_filename_full = boost::filesystem::canonical(input_filename_relative).string(); + } + metadata_file << "Input:" << input_filename_relative << endl; + metadata_file << "Input full path:" << input_filename_full << endl; + } + else + { + // Populate the metadata file + metadata_file << "Input:webcam" << endl; + } + + metadata_file << "Camera parameters:" << parameters.getFx() << "," << parameters.getFy() << "," << parameters.getCx() << "," << parameters.getCy() << endl; + + // Create the required individual recorders, CSV, HOG, aligned, video + csv_filename = filename + ".csv"; + + // Consruct HOG recorder here + if (params.outputHOG()) + { + // Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory + std::string hog_filename = filename + ".hog"; + metadata_file << "Output HOG:" << hog_filename << endl; + hog_filename = (path(record_root) / hog_filename).string(); + hog_recorder.Open(hog_filename); + } + + // saving the videos + if (params.outputTracked()) + { + if (parameters.isSequence()) + { + // Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory + this->media_filename = filename + ".avi"; + metadata_file << "Output video:" << this->media_filename << endl; + this->media_filename = (path(record_root) / this->media_filename).string(); + } + else + { + this->media_filename = filename + ".jpg"; + metadata_file << "Output image:" << this->media_filename << endl; + this->media_filename = (path(record_root) / this->media_filename).string(); + } + } + + // Prepare image recording + if (params.outputAlignedFaces()) + { + aligned_output_directory = filename + "_aligned"; + metadata_file << "Output aligned directory:" << this->aligned_output_directory << endl; + this->aligned_output_directory = (path(record_root) / this->aligned_output_directory).string(); + CreateDirectory(aligned_output_directory); + } + + observation_count = 0; + +} + RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::vector& arguments):video_writer(), params(parameters) { @@ -127,85 +208,32 @@ RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFa } } - // Construct the directories required for the output - CreateDirectory(record_root); + PrepareRecording(); +} - // Create the filename for the general output file that contains all of the meta information about the recording - path of_det_name(filename); - of_det_name = path(record_root) / path(filename + "_of_details.txt"); - - // Write in the of file what we are outputing what is the input etc. - metadata_file.open(of_det_name.string(), std::ios_base::out); - if (!metadata_file.is_open()) +RecorderOpenFace::RecorderOpenFace(const std::string in_filename, RecorderOpenFaceParameters parameters, std::string output_directory, std::string output_name) +{ + // From the filename, strip out the name without directory and extension + if (boost::filesystem::is_directory(in_filename)) { - cout << "ERROR: could not open the output file:" << of_det_name << ", either the path of the output directory is wrong or you do not have the permissions to write to it" << endl; - exit(1); - } - - // Populate relative and full path names in the meta file, unless it is a webcam - if(!params.isFromWebcam()) - { - string input_filename_relative = in_filename; - string input_filename_full = in_filename; - if (!boost::filesystem::path(input_filename_full).is_absolute()) - { - input_filename_full = boost::filesystem::canonical(input_filename_relative).string(); - } - metadata_file << "Input:" << input_filename_relative << endl; - metadata_file << "Input full path:" << input_filename_full << endl; + filename = boost::filesystem::canonical(boost::filesystem::path(in_filename)).filename().string(); } else { - // Populate the metadata file - metadata_file << "Input:webcam" << endl; + filename = boost::filesystem::path(in_filename).filename().replace_extension("").string(); } - metadata_file << "Camera parameters:" << parameters.getFx() << "," << parameters.getFy() << "," << parameters.getCx() << "," << parameters.getCy() << endl; + record_root = output_directory; + filename = output_name; - // Create the required individual recorders, CSV, HOG, aligned, video - csv_filename = filename + ".csv"; - - // Consruct HOG recorder here - if(params.outputHOG()) - { - // Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory - std::string hog_filename = filename + ".hog"; - metadata_file << "Output HOG:" << hog_filename << endl; - hog_filename = (path(record_root) / hog_filename).string(); - hog_recorder.Open(hog_filename); - } - - // saving the videos - if (params.outputTracked()) - { - if(parameters.isSequence()) - { - // Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory - this->media_filename = filename + ".avi"; - metadata_file << "Output video:" << this->media_filename << endl; - this->media_filename = (path(record_root) / this->media_filename).string(); - } - else - { - this->media_filename = filename + ".jpg"; - metadata_file << "Output image:" << this->media_filename << endl; - this->media_filename = (path(record_root) / this->media_filename).string(); - } - } - - // Prepare image recording - if (params.outputAlignedFaces()) - { - aligned_output_directory = filename + "_aligned"; - metadata_file << "Output aligned directory:" << this->aligned_output_directory << endl; - this->aligned_output_directory = (path(record_root) / this->aligned_output_directory).string(); - CreateDirectory(aligned_output_directory); - } - - observation_count = 0; + // If recording directory not set, record to default location + if (record_root.empty()) + record_root = default_record_directory; + PrepareRecording(); } + void RecorderOpenFace::SetObservationFaceAlign(const cv::Mat& aligned_face) { this->aligned_face = aligned_face; diff --git a/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp index 20b58df..7fc99d0 100644 --- a/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp +++ b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp @@ -40,8 +40,6 @@ using namespace Utilities; RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector &arguments, bool sequence, bool from_webcam, float fx, float fy, float cx, float cy, double fps_vid_out) { - string separator = string(1, boost::filesystem::path::preferred_separator); - this->is_sequence = sequence; this->is_from_webcam = from_webcam; this->fx = fx; @@ -138,3 +136,35 @@ RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector } +RecorderOpenFaceParameters::RecorderOpenFaceParameters(bool sequence, bool is_from_webcam, bool output_2D_landmarks, bool output_3D_landmarks, + bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked, + bool output_aligned_faces, float fx = -1, float fy = -1, float cx = -1, float cy = -1, double fps_vid_out = 30) +{ + this->is_sequence = sequence; + this->is_from_webcam = is_from_webcam; + this->fx = fx; + this->fy = fy; + this->cx = cx; + this->cy = cy; + + if (fps_vid_out > 0) + { + this->fps_vid_out = fps_vid_out; + } + else + { + this->fps_vid_out = 30; // If an illegal value for fps provided, default to 30 + } + // Default output code + this->output_codec = "DIVX"; + + this->output2DLandmarks = output_2D_landmarks; + this->output3DLandmarks = output_3D_landmarks; + this->outputPDMParams = output_model_params; + this->outputPose = output_pose; + this->outputAUs = output_AUs; + this->outputGaze = output_gaze; + this->outputHOG = output_hog; + this->outputTracked = output_tracked; + this->outputAlignedFaces = output_aligned_faces; +} \ No newline at end of file