More patches and small fixes for recording and visualization.

exe/FeatureExtraction/FeatureExtraction.cpp

@@ -131,6 +131,11 @@ int main (int argc, char **argv)
 
 		INFO_STREAM("Device or file opened");
 		
+		if (sequence_reader.IsWebcam())
+		{
+			INFO_STREAM("WARNING: using a webcam in feature extraction, Action Unit predictions will not be as accurate in real-time webcam mode");
+		}
+
 		cv::Mat captured_image;
 		
 		Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.fps);
@@ -171,7 +176,7 @@ int main (int argc, char **argv)
 			// Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization
 			if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog)
 			{
-				face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, false);
+				face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam());
 				face_analyser.GetLatestAlignedFace(sim_warped_img);
 				face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols);
 			}
@@ -216,6 +221,10 @@ int main (int argc, char **argv)
 			if(sequence_reader.GetProgress() >= reported_completion / 10.0)
 			{
 				cout << reported_completion * 10 << "% ";
+				if (reported_completion == 10)
+				{
+					cout << endl;
+				}
 				reported_completion = reported_completion + 1;
 			}
 

lib/local/Utilities/include/SequenceCapture.h

@@ -73,6 +73,8 @@ namespace Utilities
 		// Video file
 		bool OpenVideoFile(std::string video_file, float fx = -1, float fy = -1, float cx = -1, float cy = -1);
 
+		bool IsWebcam() { return is_webcam; }
+
 		// Getting the next frame
 		cv::Mat GetNextFrame();
 
@@ -99,6 +101,7 @@ namespace Utilities
 		// Allows to differentiate if failed because no input specified or if failed to open a specified input
 		bool no_input_specified;
 
+
 	private:
 
 		// Used for capturing webcam and video
@@ -107,12 +110,7 @@ namespace Utilities
 		// Storing the latest captures
 		cv::Mat latest_frame;
 		cv::Mat latest_gray_frame;
-
+		
-
-		// Keeping track if we are opening a video, webcam or image sequence
-		bool is_webcam;
-		bool is_image_seq;
-
 		// Keeping track of frame number and the files in the image sequence
 		size_t  frame_num;
 		std::vector<std::string> image_files;
@@ -123,6 +121,10 @@ namespace Utilities
 		// If using a webcam, helps to keep track of time
 		int64 start_time;
 
+		// Keeping track if we are opening a video, webcam or image sequence
+		bool is_webcam;
+		bool is_image_seq;
+
 		void SetCameraIntrinsics(float fx, float fy, float cx, float cy);
 
 

matlab_runners/Demos/feature_extraction_demo_vid.m

@@ -127,7 +127,7 @@ au_reg_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r'));
 
 aus = all_params(valid_frames, au_reg_inds);
 figure
-plot(time, aus);
+plot(time_stamps, aus);
 title('Facial Action Units (intensity)');
 xlabel('Time (s)');
 ylabel('Intensity');

matlab_runners/Demos/gaze_extraction_demo_vid.m

@@ -6,37 +6,12 @@ else
     executable = '"../../x64/Release/FeatureExtraction.exe"';
 end
 
-output = './output_features_vid/';
+output = './processed_features/';
+    
+in_file = '../../samples/2015-10-15-15-14.avi';
 
-if(~exist(output, 'file'))
+command = sprintf('%s -f "%s" -out_dir "%s" -gaze -verbose', executable, in_file, output);
-    mkdir(output)
-end
     
-in_files = dir('../../samples/2015-10-15-15-14.avi');
-% some parameters
-verbose = true;
-
-command = executable;
-command = cat(2, command, ' -verbose  -no2Dfp -no3Dfp -noMparams -noPose -noAUs ');
-
-% add all videos to single argument list (so as not to load the model anew
-% for every video)
-for i=1:numel(in_files)
-    
-    inputFile = ['../../samples/', in_files(i).name];
-    [~, name, ~] = fileparts(inputFile);
-    
-    % where to output tracking results
-    outputFile_gaze = [output name '_gaze.txt'];
-    
-    if(~exist([output name], 'file'))
-        mkdir([output name]);
-    end
-    
-    command = cat(2, command, ['-fx 700 -fy 700 -f "' inputFile '" -of "' outputFile_gaze '"']);
-               
-end
-
 if(isunix)
     unix(command);
 else
@@ -44,25 +19,23 @@ else
 end
 
 %% Demonstrating reading the output files
-filename = [output name];
+[~, out_filename,~] = fileparts(in_file);
+out_filename = sprintf("%s/%s.csv", output, out_filename);
 
-% Read gaze (x,y,z) for one eye and (x,y,z) for another
+% First read in the column names
-gaze  = dlmread([filename, '_gaze.txt'], ',', 1, 0);
+tab = readtable(out_filename);
+column_names = tab.Properties.VariableNames;
 
-% This indicates which frames were succesfully tracked
+all_params  = dlmread(out_filename, ',', 1, 0);
-valid_frames = gaze(:,4);
 
-% only picking left, right and up down views for visualisation
+gaze_angle_ids = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle_'));
-gaze = gaze(:,[5,6,7,8,9,10]);
+
-gaze = (gaze(:,[1,2,3]) + gaze(:,[4,5,6]))/2;
+gaze = all_params(:,gaze_angle_ids);
-gaze(:,1) = smooth(gaze(:,1));
-gaze(:,2) = smooth(gaze(:,2));
-gaze(:,3) = smooth(gaze(:,3));
 
 plot(gaze(:,1), 'DisplayName', 'Left - right');
 hold on;
 plot(gaze(:,2), 'DisplayName', 'Up - down');
 xlabel('Frame') % x-axis label
-ylabel('Gaze vector size') % y-axis label
+ylabel('Angle in radians') % y-axis label
 legend('show');
 hold off;

matlab_runners/Full_test_suite.m

@@ -52,6 +52,7 @@ cd('Demos');
 run_demo_images;
 run_demo_videos;
 run_demo_video_multi;
+run_demo_align_size;
 feature_extraction_demo_vid;
 feature_extraction_demo_img_seq;
 gaze_extraction_demo_vid;