From 21bdf03801b2065f1a1b192728b9a6f88a2e9467 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Fri, 24 Nov 2017 17:16:15 +0000 Subject: [PATCH] More patches and small fixes for recording and visualization. --- exe/FeatureExtraction/FeatureExtraction.cpp | 11 +++- lib/local/Utilities/include/SequenceCapture.h | 14 +++-- .../Demos/feature_extraction_demo_vid.m | 2 +- .../Demos/gaze_extraction_demo_vid.m | 55 +++++-------------- matlab_runners/Full_test_suite.m | 1 + 5 files changed, 34 insertions(+), 49 deletions(-) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index b507602..a018731 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -131,6 +131,11 @@ int main (int argc, char **argv) INFO_STREAM("Device or file opened"); + if (sequence_reader.IsWebcam()) + { + INFO_STREAM("WARNING: using a webcam in feature extraction, Action Unit predictions will not be as accurate in real-time webcam mode"); + } + cv::Mat captured_image; Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.fps); @@ -171,7 +176,7 @@ int main (int argc, char **argv) // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog) { - face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, false); + face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam()); face_analyser.GetLatestAlignedFace(sim_warped_img); face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); } @@ -216,6 +221,10 @@ int main (int argc, char **argv) if(sequence_reader.GetProgress() >= reported_completion / 10.0) { cout << reported_completion * 10 << "% "; + if (reported_completion == 10) + { + cout << endl; + } reported_completion = reported_completion + 1; } diff --git a/lib/local/Utilities/include/SequenceCapture.h b/lib/local/Utilities/include/SequenceCapture.h index 1b42143..8995b7b 100644 --- a/lib/local/Utilities/include/SequenceCapture.h +++ b/lib/local/Utilities/include/SequenceCapture.h @@ -73,6 +73,8 @@ namespace Utilities // Video file bool OpenVideoFile(std::string video_file, float fx = -1, float fy = -1, float cx = -1, float cy = -1); + bool IsWebcam() { return is_webcam; } + // Getting the next frame cv::Mat GetNextFrame(); @@ -99,6 +101,7 @@ namespace Utilities // Allows to differentiate if failed because no input specified or if failed to open a specified input bool no_input_specified; + private: // Used for capturing webcam and video @@ -107,12 +110,7 @@ namespace Utilities // Storing the latest captures cv::Mat latest_frame; cv::Mat latest_gray_frame; - - - // Keeping track if we are opening a video, webcam or image sequence - bool is_webcam; - bool is_image_seq; - + // Keeping track of frame number and the files in the image sequence size_t frame_num; std::vector image_files; @@ -123,6 +121,10 @@ namespace Utilities // If using a webcam, helps to keep track of time int64 start_time; + // Keeping track if we are opening a video, webcam or image sequence + bool is_webcam; + bool is_image_seq; + void SetCameraIntrinsics(float fx, float fy, float cx, float cy); diff --git a/matlab_runners/Demos/feature_extraction_demo_vid.m b/matlab_runners/Demos/feature_extraction_demo_vid.m index 0f01c5f..ccdb85a 100644 --- a/matlab_runners/Demos/feature_extraction_demo_vid.m +++ b/matlab_runners/Demos/feature_extraction_demo_vid.m @@ -127,7 +127,7 @@ au_reg_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r')); aus = all_params(valid_frames, au_reg_inds); figure -plot(time, aus); +plot(time_stamps, aus); title('Facial Action Units (intensity)'); xlabel('Time (s)'); ylabel('Intensity'); diff --git a/matlab_runners/Demos/gaze_extraction_demo_vid.m b/matlab_runners/Demos/gaze_extraction_demo_vid.m index ea747d0..574f299 100644 --- a/matlab_runners/Demos/gaze_extraction_demo_vid.m +++ b/matlab_runners/Demos/gaze_extraction_demo_vid.m @@ -6,37 +6,12 @@ else executable = '"../../x64/Release/FeatureExtraction.exe"'; end -output = './output_features_vid/'; +output = './processed_features/'; + +in_file = '../../samples/2015-10-15-15-14.avi'; -if(~exist(output, 'file')) - mkdir(output) -end +command = sprintf('%s -f "%s" -out_dir "%s" -gaze -verbose', executable, in_file, output); -in_files = dir('../../samples/2015-10-15-15-14.avi'); -% some parameters -verbose = true; - -command = executable; -command = cat(2, command, ' -verbose -no2Dfp -no3Dfp -noMparams -noPose -noAUs '); - -% add all videos to single argument list (so as not to load the model anew -% for every video) -for i=1:numel(in_files) - - inputFile = ['../../samples/', in_files(i).name]; - [~, name, ~] = fileparts(inputFile); - - % where to output tracking results - outputFile_gaze = [output name '_gaze.txt']; - - if(~exist([output name], 'file')) - mkdir([output name]); - end - - command = cat(2, command, ['-fx 700 -fy 700 -f "' inputFile '" -of "' outputFile_gaze '"']); - -end - if(isunix) unix(command); else @@ -44,25 +19,23 @@ else end %% Demonstrating reading the output files -filename = [output name]; +[~, out_filename,~] = fileparts(in_file); +out_filename = sprintf("%s/%s.csv", output, out_filename); -% Read gaze (x,y,z) for one eye and (x,y,z) for another -gaze = dlmread([filename, '_gaze.txt'], ',', 1, 0); +% First read in the column names +tab = readtable(out_filename); +column_names = tab.Properties.VariableNames; -% This indicates which frames were succesfully tracked -valid_frames = gaze(:,4); +all_params = dlmread(out_filename, ',', 1, 0); -% only picking left, right and up down views for visualisation -gaze = gaze(:,[5,6,7,8,9,10]); -gaze = (gaze(:,[1,2,3]) + gaze(:,[4,5,6]))/2; -gaze(:,1) = smooth(gaze(:,1)); -gaze(:,2) = smooth(gaze(:,2)); -gaze(:,3) = smooth(gaze(:,3)); +gaze_angle_ids = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle_')); + +gaze = all_params(:,gaze_angle_ids); plot(gaze(:,1), 'DisplayName', 'Left - right'); hold on; plot(gaze(:,2), 'DisplayName', 'Up - down'); xlabel('Frame') % x-axis label -ylabel('Gaze vector size') % y-axis label +ylabel('Angle in radians') % y-axis label legend('show'); hold off; \ No newline at end of file diff --git a/matlab_runners/Full_test_suite.m b/matlab_runners/Full_test_suite.m index 877fdc4..d91f631 100644 --- a/matlab_runners/Full_test_suite.m +++ b/matlab_runners/Full_test_suite.m @@ -52,6 +52,7 @@ cd('Demos'); run_demo_images; run_demo_videos; run_demo_video_multi; +run_demo_align_size; feature_extraction_demo_vid; feature_extraction_demo_img_seq; gaze_extraction_demo_vid;