r/sustaining_gazes
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Getting ready to testing a new sequence recorder.

Browse source
This commit is contained in:
Tadas Baltrusaitis 2017-11-09 19:56:16 +00:00
parent 8ded025baf
commit e22060d073
4 changed files with 70 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
exe/FeatureExtraction/FeatureExtraction.cpp
View file

@ -98,8 +98,13 @@ vector<string> get_arguments(int argc, char **argv)
void get_visualization_params(bool& visualize_track, bool& visualize_align, bool& visualize_hog, vector<string> &arguments);
// Some globals for tracking timing information for visualisation (TODO bit ugly)
double fps_tracker = -1.0;
int64 t0 = 0;
int frame_count = 0;
// Visualising the results TODO separate class
void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, int frame_count, double fx, double fy, double cx, double cy)
void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& face_model, const LandmarkDetector::FaceModelParameters& det_parameters, cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, double fx, double fy, double cx, double cy)
{
// Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised
@ -136,20 +141,20 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f
}
// Work out the framerate TODO
//if (frame_count % 10 == 0)
//{
// double t1 = cv::getTickCount();
// fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency());
// t0 = t1;
//}
//// Write out the framerate on the image before displaying it
//char fpsC[255];
//std::sprintf(fpsC, "%d", (int)fps_tracker);
//string fpsSt("FPS:");
//fpsSt += fpsC;
//cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA);
if (frame_count % 10 == 0)
{
double t1 = cv::getTickCount();
fps_tracker = 10.0 / (double(t1 - t0) / cv::getTickFrequency());
t0 = t1;
}
// Write out the framerate on the image before displaying it
char fpsC[255];
std::sprintf(fpsC, "%d", (int)fps_tracker);
string fpsSt("FPS:");
fpsSt += fpsC;
cv::putText(captured_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA);
frame_count++;
}
int main (int argc, char **argv)
@ -190,10 +195,11 @@ int main (int argc, char **argv)
Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.fps);
Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments);
int frame_count = 0;
captured_image = sequence_reader.GetNextFrame();
// For reporting progress
double reported_completion = 0;
INFO_STREAM("Starting tracking");
while (!captured_image.empty())
{
@ -224,7 +230,7 @@ int main (int argc, char **argv)
// As this can be expensive only compute it if needed by output or visualization
if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualize_align || visualize_hog)
{
face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, time_stamp, false, !det_parameters.quiet_mode);
face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, false, !det_parameters.quiet_mode);
face_analyser.GetLatestAlignedFace(sim_warped_img);
if (!det_parameters.quiet_mode && visualize_align)
@ -250,7 +256,7 @@ int main (int argc, char **argv)
// Drawing the visualization on the captured image
if (recording_params.outputTrackedVideo() || (visualize_track && !det_parameters.quiet_mode))
{
visualise_tracking(captured_image, face_model, det_parameters, gazeDirection0, gazeDirection1, frame_count, fx, fy, cx, cy);
visualise_tracking(captured_image, face_model, det_parameters, gazeDirection0, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
}
// Setting up the recorder output
@ -258,9 +264,10 @@ int main (int argc, char **argv)
open_face_rec.SetObservationVisualization(captured_image);
open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model));
open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(fx, fy, cx, cy), face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success);
open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy),
face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success);
open_face_rec.SetObservationPose(pose_estimate);
open_face_rec.SetObservationTimestamp(time_stamp);
open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp);
open_face_rec.WriteObservation();
// Visualize the image if desired
@ -290,20 +297,18 @@ int main (int argc, char **argv)
}
}
if(total_frames != -1)
if(sequence_reader.GetProgress() >= reported_completion / 10.0)
{
if((double)frame_count/(double)total_frames >= reported_completion / 10.0)
{
cout << reported_completion * 10 << "% ";
reported_completion = reported_completion + 1;
}
cout << reported_completion * 10 << "% ";
reported_completion = reported_completion + 1;
}
}
open_face_rec.Close();
if (output_files.size() > 0 && recording_params.outputAUs())
if (recording_params.outputAUs())
{
cout << "Postprocessing the Action Unit predictions" << endl;
face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile());

5
lib/local/Utilities/include/SequenceCapture.h
View file

@ -79,6 +79,7 @@ namespace Utilities
// Getting the most recent grayscale frame (need to call GetNextFrame first)
cv::Mat_<uchar> GetGrayFrame();
// Parameters describing the sequence and it's progress
double GetProgress();
bool IsOpened();
@ -90,6 +91,8 @@ namespace Utilities
double fps;
double time_stamp;
// Name of the video file, image directory, or the webcam
std::string name;
@ -116,6 +119,8 @@ namespace Utilities
bool img_grabbed;
// If using a webcam, helps to keep track of time
int64 start_time;
};
}
#endif

2
lib/local/Utilities/src/RecorderOpenFace.cpp
View file

@ -157,7 +157,7 @@ void RecorderOpenFace::WriteObservation()
{
// As we are writing out the header, work out some things like number of landmarks, names of AUs etc.
int num_face_landmarks = landmarks_2D.rows / 2;
int num_eye_landmarks = eye_landmarks.size();
int num_eye_landmarks = (int)eye_landmarks.size();
int num_model_modes = pdm_params_local.rows;
std::vector<std::string> au_names_class;

50
lib/local/Utilities/src/SequenceCapture.cpp
View file

@ -142,7 +142,7 @@ bool SequenceCapture::Open(std::vector<std::string> arguments)
}
}
for (int i = arguments.size() - 1; i >= 0; --i)
for (size_t i = arguments.size() - 1; i >= 0; --i)
{
if (!valid[i])
{
@ -190,8 +190,8 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height,
vid_length = 0;
frame_num = 0;
this->frame_width = capture.get(CV_CAP_PROP_FRAME_WIDTH);
this->frame_height = capture.get(CV_CAP_PROP_FRAME_HEIGHT);
this->frame_width = (int)capture.get(CV_CAP_PROP_FRAME_WIDTH);
this->frame_height = (int)capture.get(CV_CAP_PROP_FRAME_HEIGHT);
if (!capture.isOpened())
{
@ -222,14 +222,16 @@ bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height,
// Use a rough guess-timate of focal length
if (fx == -1)
{
fx = 500 * (frame_width / 640.0);
fy = 500 * (frame_height / 480.0);
fx = (float)(500.0 * (frame_width / 640.0));
fy = (float)(500.0 * (frame_height / 480.0));
fx = (fx + fy) / 2.0;
fx = (fx + fy) / 2.0f;
fy = fx;
}
this->name = "webcam"; // TODO number
start_time = cv::getTickCount();
return true;
@ -258,10 +260,10 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy,
is_webcam = false;
is_image_seq = false;
this->frame_width = capture.get(CV_CAP_PROP_FRAME_WIDTH);
this->frame_height = capture.get(CV_CAP_PROP_FRAME_HEIGHT);
this->frame_width = (int)capture.get(CV_CAP_PROP_FRAME_WIDTH);
this->frame_height = (int)capture.get(CV_CAP_PROP_FRAME_HEIGHT);
vid_length = capture.get(CV_CAP_PROP_FRAME_COUNT);
vid_length = (int)capture.get(CV_CAP_PROP_FRAME_COUNT);
frame_num = 0;
if (capture.isOpened())
@ -279,14 +281,14 @@ bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy,
// Use a rough guess-timate of focal length
if (fx == -1)
{
fx = 500 * (frame_width / 640.0);
fy = 500 * (frame_height / 480.0);
fx = 500.0f * (frame_width / 640.0f);
fy = 500.0f * (frame_height / 480.0f);
fx = (fx + fy) / 2.0;
fx = (fx + fy) / 2.0f;
fy = fx;
}
this->name = boost::filesystem::path(video_file).filename.replace_extension("").string();
this->name = boost::filesystem::path(video_file).filename().replace_extension("").string();
return true;
@ -338,17 +340,17 @@ bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float f
// Use a rough guess-timate of focal length
if (fx == -1)
{
fx = 500 * (frame_width / 640.0);
fy = 500 * (frame_height / 480.0);
fx = 500.0f * (frame_width / 640.0f);
fy = 500.0f * (frame_height / 480.0f);
fx = (fx + fy) / 2.0;
fx = (fx + fy) / 2.0f;
fy = fx;
}
// No fps as we have a sequence
this->fps = 0;
this->name = boost::filesystem::path(directory).filename.string();
this->name = boost::filesystem::path(directory).filename().string();
return true;
@ -400,7 +402,7 @@ cv::Mat SequenceCapture::GetNextFrame()
{
frame_num++;
if (is_webcam && !is_image_seq)
if (is_webcam || !is_image_seq)
{
bool success = capture.read(latest_frame);
@ -410,6 +412,17 @@ cv::Mat SequenceCapture::GetNextFrame()
// Indicate lack of success by returning an empty image
latest_frame = cv::Mat();
}
// Recording the timestamp
if (!is_webcam)
{
time_stamp = frame_num * (1.0 / fps);
}
else
{
time_stamp = (cv::getTickCount() - start_time) / cv::getTickFrequency();
}
}
else if (is_image_seq)
{
@ -420,6 +433,7 @@ cv::Mat SequenceCapture::GetNextFrame()
}
latest_frame = cv::imread(image_files[frame_num-1], -1);
time_stamp = 0;
}
// Set the grayscale frame