r/sustaining_gazes
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Getting ready to testing a new sequence recorder.

Browse source
This commit is contained in:
Tadas Baltrusaitis 2017-11-09 19:56:16 +00:00
parent 8ded025baf
commit e22060d073
4 changed files with 70 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
exe/FeatureExtraction/FeatureExtraction.cpp
View file

@ -98,8 +98,13 @@ vector<string> get_arguments(int argc, char **argv)
void get_visualization_params(bool& visualize_track, bool& visualize_align, bool& visualize_hog, vector<string> &arguments); void get_visualization_params(bool& visualize_track, bool& visualize_align, bool& visualize_hog, vector<string> &arguments);
// Some globals for tracking timing information for visualisation (TODO bit ugly)
double fps_tracker = -1.0;
int64 t0 = 0;
int frame_count = 0;
// Visualising the results TODO separate class // Visualising the results TODO separate class
void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, int frame_count, double fx, double fy, double cx, double cy) void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, double fx, double fy, double cx, double cy)
{ {
// Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised
@ -136,20 +141,20 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
} }
// Work out the framerate TODO // Work out the framerate TODO
//if (frame_count % 10 == 0) if (frame_count % 10 == 0)
//{ {
// double t1 = cv::getTickCount(); double t1 = cv::getTickCount();
// fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency()); fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency());
// t0 = t1; t0 = t1;
//} }
//// Write out the framerate on the image before displaying it
//char fpsC[255];
//std::sprintf(fpsC, "%d", (int)fps_tracker);
//string fpsSt("FPS:");
//fpsSt += fpsC;
//cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA);
// Write out the framerate on the image before displaying it
char fpsC[255];
std::sprintf(fpsC, "%d", (int)fps_tracker);
string fpsSt("FPS:");
fpsSt += fpsC;
cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA);
frame_count++;
} }
int main (int argc, char **argv) int main (int argc, char **argv)
@ -190,10 +195,11 @@ int main (int argc, char **argv)
Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.fps); Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.fps);
Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments);
int frame_count = 0;
captured_image = sequence_reader.GetNextFrame(); captured_image = sequence_reader.GetNextFrame();
// For reporting progress
double reported_completion = 0;
INFO_STREAM("Starting tracking"); INFO_STREAM("Starting tracking");
while (!captured_image.empty()) while (!captured_image.empty())
{ {
@ -224,7 +230,7 @@ int main (int argc, char **argv)
// As this can be expensive only compute it if needed by output or visualization // As this can be expensive only compute it if needed by output or visualization
if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualize_align || visualize_hog) if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualize_align || visualize_hog)
{ {
face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, time_stamp, false, !det_parameters.quiet_mode); face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, false, !det_parameters.quiet_mode);
face_analyser.GetLatestAlignedFace(sim_warped_img); face_analyser.GetLatestAlignedFace(sim_warped_img);
if (!det_parameters.quiet_mode && visualize_align) if (!det_parameters.quiet_mode && visualize_align)
@ -250,7 +256,7 @@ int main (int argc, char **argv)
// Drawing the visualization on the captured image // Drawing the visualization on the captured image
if (recording_params.outputTrackedVideo() || (visualize_track && !det_parameters.quiet_mode)) if (recording_params.outputTrackedVideo() || (visualize_track && !det_parameters.quiet_mode))
{ {
visualise_tracking(captured_image, face_model, det_parameters, gazeDirection0, gazeDirection1, frame_count, fx, fy, cx, cy); visualise_tracking(captured_image, face_model, det_parameters, gazeDirection0, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
} }
// Setting up the recorder output // Setting up the recorder output
@ -258,9 +264,10 @@ int main (int argc, char **argv)
open_face_rec.SetObservationVisualization(captured_image); open_face_rec.SetObservationVisualization(captured_image);
open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model)); open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model));
open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(fx, fy, cx, cy), face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success); open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy),
face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success);
open_face_rec.SetObservationPose(pose_estimate); open_face_rec.SetObservationPose(pose_estimate);
open_face_rec.SetObservationTimestamp(time_stamp); open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp);
open_face_rec.WriteObservation(); open_face_rec.WriteObservation();
// Visualize the image if desired // Visualize the image if desired
@ -290,20 +297,18 @@ int main (int argc, char **argv)
} }
} }
if(total_frames != -1)
{ if(sequence_reader.GetProgress() >= reported_completion / 10.0)
if((double)frame_count/(double)total_frames >= reported_completion / 10.0)
{ {
cout << reported_completion * 10 << "% "; cout << reported_completion * 10 << "% ";
reported_completion = reported_completion + 1; reported_completion = reported_completion + 1;
} }
}
} }
open_face_rec.Close(); open_face_rec.Close();
if (output_files.size() > 0 && recording_params.outputAUs()) if (recording_params.outputAUs())
{ {
cout << "Postprocessing the Action Unit predictions" << endl; cout << "Postprocessing the Action Unit predictions" << endl;
face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile()); face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile());

5
lib/local/Utilities/include/SequenceCapture.h
View file

@ -79,6 +79,7 @@ namespace Utilities
// Getting the most recent grayscale frame (need to call GetNextFrame first) // Getting the most recent grayscale frame (need to call GetNextFrame first)
cv::Mat_<uchar> GetGrayFrame(); cv::Mat_<uchar> GetGrayFrame();
// Parameters describing the sequence and it's progress
double GetProgress(); double GetProgress();
bool IsOpened(); bool IsOpened();
@ -90,6 +91,8 @@ namespace Utilities
double fps; double fps;
double time_stamp;
// Name of the video file, image directory, or the webcam // Name of the video file, image directory, or the webcam
std::string name; std::string name;
@ -116,6 +119,8 @@ namespace Utilities
bool img_grabbed; bool img_grabbed;
// If using a webcam, helps to keep track of time
int64 start_time;
}; };
} }
#endif #endif

2
lib/local/Utilities/src/RecorderOpenFace.cpp
View file

@ -157,7 +157,7 @@ void RecorderOpenFace::WriteObservation()
{ {
// As we are writing out the header, work out some things like number of landmarks, names of AUs etc. // As we are writing out the header, work out some things like number of landmarks, names of AUs etc.
int num_face_landmarks = landmarks_2D.rows / 2; int num_face_landmarks = landmarks_2D.rows / 2;
int num_eye_landmarks = eye_landmarks.size(); int num_eye_landmarks = (int)eye_landmarks.size();
int num_model_modes = pdm_params_local.rows; int num_model_modes = pdm_params_local.rows;
std::vector<std::string> au_names_class; std::vector<std::string> au_names_class;

50
lib/local/Utilities/src/SequenceCapture.cpp
View file

@ -142,7 +142,7 @@ bool SequenceCapture::Open(std::vector<std::string> arguments)
} }
} }
for (int i = arguments.size() - 1; i >= 0; --i) for (size_t i = arguments.size() - 1; i >= 0; --i)
{ {
if (!valid[i]) if (!valid[i])
{ {
@ -190,8 +190,8 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height,
vid_length = 0; vid_length = 0;
frame_num = 0; frame_num = 0;
this->frame_width = capture.get(CV_CAP_PROP_FRAME_WIDTH); this->frame_width = (int)capture.get(CV_CAP_PROP_FRAME_WIDTH);
this->frame_height = capture.get(CV_CAP_PROP_FRAME_HEIGHT); this->frame_height = (int)capture.get(CV_CAP_PROP_FRAME_HEIGHT);
if (!capture.isOpened()) if (!capture.isOpened())
{ {
@ -222,15 +222,17 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height,
// Use a rough guess-timate of focal length // Use a rough guess-timate of focal length
if (fx == -1) if (fx == -1)
{ {
fx = 500 * (frame_width / 640.0); fx = (float)(500.0 * (frame_width / 640.0));
fy = 500 * (frame_height / 480.0); fy = (float)(500.0 * (frame_height / 480.0));
fx = (fx + fy) / 2.0; fx = (fx + fy) / 2.0f;
fy = fx; fy = fx;
} }
this->name = "webcam"; // TODO number this->name = "webcam"; // TODO number
start_time = cv::getTickCount();
return true; return true;
} }
@ -258,10 +260,10 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy,
is_webcam = false; is_webcam = false;
is_image_seq = false; is_image_seq = false;
this->frame_width = capture.get(CV_CAP_PROP_FRAME_WIDTH); this->frame_width = (int)capture.get(CV_CAP_PROP_FRAME_WIDTH);
this->frame_height = capture.get(CV_CAP_PROP_FRAME_HEIGHT); this->frame_height = (int)capture.get(CV_CAP_PROP_FRAME_HEIGHT);
vid_length = capture.get(CV_CAP_PROP_FRAME_COUNT); vid_length = (int)capture.get(CV_CAP_PROP_FRAME_COUNT);
frame_num = 0; frame_num = 0;
if (capture.isOpened()) if (capture.isOpened())
@ -279,14 +281,14 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy,
// Use a rough guess-timate of focal length // Use a rough guess-timate of focal length
if (fx == -1) if (fx == -1)
{ {
fx = 500 * (frame_width / 640.0); fx = 500.0f * (frame_width / 640.0f);
fy = 500 * (frame_height / 480.0); fy = 500.0f * (frame_height / 480.0f);
fx = (fx + fy) / 2.0; fx = (fx + fy) / 2.0f;
fy = fx; fy = fx;
} }
this->name = boost::filesystem::path(video_file).filename.replace_extension("").string(); this->name = boost::filesystem::path(video_file).filename().replace_extension("").string();
return true; return true;
@ -338,17 +340,17 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f
// Use a rough guess-timate of focal length // Use a rough guess-timate of focal length
if (fx == -1) if (fx == -1)
{ {
fx = 500 * (frame_width / 640.0); fx = 500.0f * (frame_width / 640.0f);
fy = 500 * (frame_height / 480.0); fy = 500.0f * (frame_height / 480.0f);
fx = (fx + fy) / 2.0; fx = (fx + fy) / 2.0f;
fy = fx; fy = fx;
} }
// No fps as we have a sequence // No fps as we have a sequence
this->fps = 0; this->fps = 0;
this->name = boost::filesystem::path(directory).filename.string(); this->name = boost::filesystem::path(directory).filename().string();
return true; return true;
@ -400,7 +402,7 @@ cv::Mat SequenceCapture::GetNextFrame()
{ {
frame_num++; frame_num++;
if (is_webcam && !is_image_seq) if (is_webcam || !is_image_seq)
{ {
bool success = capture.read(latest_frame); bool success = capture.read(latest_frame);
@ -410,6 +412,17 @@ cv::Mat SequenceCapture::GetNextFrame()
// Indicate lack of success by returning an empty image // Indicate lack of success by returning an empty image
latest_frame = cv::Mat(); latest_frame = cv::Mat();
} }
// Recording the timestamp
if (!is_webcam)
{
time_stamp = frame_num * (1.0 / fps);
}
else
{
time_stamp = (cv::getTickCount() - start_time) / cv::getTickFrequency();
}
} }
else if (is_image_seq) else if (is_image_seq)
{ {
@ -420,6 +433,7 @@ cv::Mat SequenceCapture::GetNextFrame()
} }
latest_frame = cv::imread(image_files[frame_num-1], -1); latest_frame = cv::imread(image_files[frame_num-1], -1);
time_stamp = 0;
} }
// Set the grayscale frame // Set the grayscale frame