Some work on demo scripts and according bug fixes

lib/local/Utilities/src/RecorderOpenFaceParameters.cpp

@@ -44,8 +44,14 @@ RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector<std::string>
 
 	this->is_sequence = sequence;
 
-	this->fps_vid_out = fps_vid_out;
+	if(fps_vid_out > 0)
-
+	{
+		this->fps_vid_out = fps_vid_out;
+	}
+	else
+	{
+		this->fps_vid_out = 30; // If an illegal value for fps provided, default to 30
+	}
 	// Default output code
 	this->output_codec = "DIVX";
 

lib/local/Utilities/src/SequenceCapture.cpp

@@ -403,7 +403,6 @@ void SequenceCapture::SetCameraIntrinsics(float fx, float fy, float cx, float cy
 
 cv::Mat SequenceCapture::GetNextFrame()
 {
-	frame_num++;
 
 	if (is_webcam || !is_image_seq)
 	{
@@ -429,19 +428,23 @@ cv::Mat SequenceCapture::GetNextFrame()
 	}
 	else if (is_image_seq)
 	{
-		if (image_files.empty() || frame_num - 1 > image_files.size())
+		if (image_files.empty() || frame_num >= image_files.size())
 		{
 			// Indicate lack of success by returning an empty image
 			latest_frame = cv::Mat();
 		}
-
+		else
-		latest_frame = cv::imread(image_files[frame_num-1], -1);
+		{
+			latest_frame = cv::imread(image_files[frame_num], -1);			
+		}
 		time_stamp = 0;
 	}
 
 	// Set the grayscale frame
 	convertToGrayscale(latest_frame, latest_gray_frame);
 
+	frame_num++;
+
 	return latest_frame;
 }
 

matlab_runners/Demos/feature_extraction_demo_img_seq.m

@@ -1,44 +1,25 @@
+% A demo script that demonstrates how to process a single video file using
+% OpenFace and extract and visualize all of the features
+
 clear
 
+% The location executable will depend on the OS
 if(isunix)
     executable = '"../../build/bin/FeatureExtraction"';
 else
     executable = '"../../x64/Release/FeatureExtraction.exe"';
 end
 
-output = './output_features_seq/';
+% Input file
+in_dir = '../../image_sequence';
 
-if(~exist(output, 'file'))
+% Where to store the output
-    mkdir(output)
+output_dir = './processed_features/';
-end
-    
-in_dirs = {'../../image_sequence'};
-% some parameters
-verbose = true;
 
-command = executable;
+% This will take directory after -fdir and output all the features to directory
-
+% after -out_dir
-% Remove for a speedup
+command = sprintf('%s -fdir "%s" -out_dir "%s" -verbose', executable, in_dir, output_dir);
-command = cat(2, command, ' -verbose ');
-
-% add all videos to single argument list (so as not to load the model anew
-% for every video)
-for i=1:numel(in_dirs)
-    
-    [~, name, ~] = fileparts(in_dirs{i});
-    
-    % where to output tracking results
-    outputFile = [output name '.txt'];
-    outputDir_aligned = [output name];
-            
-    outputHOG_aligned = [output name '.hog'];
-    
-    command = cat(2, command, ['-asvid -fdir "' in_dirs{i} '" -of "' outputFile '" ']);
-        
-    command = cat(2, command, [' -simalign "' outputDir_aligned '" -simsize 200 -hogalign "' outputHOG_aligned '"']);    
                  
-end
-
 if(isunix)
     unix(command);
 else
@@ -47,15 +28,30 @@ end
 
 %% Demonstrating reading the output files
 
-% First read in the column names
+% Most of the features will be in the csv file in the output directory with
-tab = readtable(outputFile);
+% the same name as the input file
+[~,name,~] = fileparts(in_file);
+output_csv = sprintf('%s/%s.csv', output_dir, name);
+
+% First read in the column names, to know which columns to read for
+% particular features
+tab = readtable(output_csv);
 column_names = tab.Properties.VariableNames;
 
-all_params  = dlmread(outputFile, ',', 1, 0);
+% Read all of the data
+all_params  = dlmread(output_csv, ',', 1, 0);
 
 % This indicates which frames were succesfully tracked
-valid_frames = logical(all_params(:,4));
+
-time = all_params(valid_frames, 2);
+% Find which column contains success of tracking data and timestamp data
+valid_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'success'));
+time_stamp_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'timestamp'));
+
+% Extract tracking success data and only read those frame
+valid_frames = logical(all_params(:,valid_ind));
+
+% Get the timestamp data
+time_stamps = all_params(valid_frames, time_stamp_ind);
 
 %% Finding which header line starts with p_ (basically model params)
 shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_'));
@@ -64,7 +60,7 @@ shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_'));
 shape_params  = all_params(valid_frames, shape_inds);
 
 figure
-plot(time, shape_params);
+plot(time_stamps, shape_params);
 title('Shape parameters');
 xlabel('Time (s)');
 
@@ -75,10 +71,20 @@ landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'y_'))
 xs = all_params(valid_frames, landmark_inds_x);
 ys = all_params(valid_frames, landmark_inds_y);
 
+eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_x_'));
+eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_y_'));
+
+eye_xs = all_params(valid_frames, eye_landmark_inds_x);
+eye_ys = all_params(valid_frames, eye_landmark_inds_y);
+
 figure
 
 for j = 1:size(xs,1)
     plot(xs(j,:), -ys(j,:), '.');
+    hold on;
+    plot(eye_xs(j,:), -eye_ys(j,:), '.r');
+    hold off;
+    
     xlim([min(xs(1,:)) * 0.5, max(xs(2,:))*1.4]);
     ylim([min(-ys(1,:)) * 1.4, max(-ys(2,:))*0.5]);
     xlabel('x (px)');
@@ -96,9 +102,20 @@ xs = all_params(valid_frames, landmark_inds_x);
 ys = all_params(valid_frames, landmark_inds_y);
 zs = all_params(valid_frames, landmark_inds_z);
 
+eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_X_'));
+eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_Y_'));
+eye_landmark_inds_z = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_Z_'));
+
+eye_xs = all_params(valid_frames, eye_landmark_inds_x);
+eye_ys = all_params(valid_frames, eye_landmark_inds_y);
+eye_zs = all_params(valid_frames, eye_landmark_inds_z);
+
 figure
 for j = 1:size(xs,1)
     plot3(xs(j,:), ys(j,:), zs(j,:), '.');axis equal;
+    hold on;
+    plot3(eye_xs(j,:), eye_ys(j,:), eye_zs(j,:), '.r');
+    hold off;
     xlabel('X (mm)');
     ylabel('Y (mm)');    
     zlabel('Z (mm)');    
@@ -110,7 +127,7 @@ au_reg_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r'));
 
 aus = all_params(valid_frames, au_reg_inds);
 figure
-plot(aus);
+plot(time, aus);
 title('Facial Action Units (intensity)');
 xlabel('Time (s)');
 ylabel('Intensity');
@@ -120,7 +137,7 @@ au_class_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_c'));
 
 aus = all_params(valid_frames, au_class_inds);
 figure
-plot(aus);
+plot(time, aus);
 title('Facial Action Units (presense)');
 xlabel('Time (s)');
 ylim([0,2]);
@@ -133,14 +150,30 @@ plot(pose);
 title('Pose (rotation and translation)');
 xlabel('Time (s)');
 
+%% Demo gaze
+gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle'));
+
+% Read gaze (x,y,z) for one eye and (x,y,z) for another
+gaze = all_params(valid_frames, gaze_inds);
+
+plot(time, gaze(:,1), 'DisplayName', 'Left - right');
+hold on;
+plot(time, gaze(:,2), 'DisplayName', 'Up - down');
+xlabel('Time(s)') % x-axis label
+ylabel('Angle radians') % y-axis label
+legend('show');
+hold off;
+
 %% Output HOG files
-[hog_data, valid_inds] = Read_HOG_file(outputHOG_aligned);
+output_hog_file = sprintf('%s/%s.hog', output_dir, name);
+[hog_data, valid_inds] = Read_HOG_file(output_hog_file);
 
 %% Output aligned images
-img_files = dir([outputDir_aligned, '/*.png']);
+output_aligned_dir = sprintf('%s/%s_aligned/', output_dir, name);
+img_files = dir([output_aligned_dir, '/*.bmp']);
 imgs = cell(numel(img_files, 1));
 for i=1:numel(img_files)
-   imgs{i} = imread([ outputDir_aligned, '/', img_files(i).name]);
+   imgs{i} = imread([ output_aligned_dir, '/', img_files(i).name]);
    imshow(imgs{i})
    drawnow
 end

matlab_runners/Demos/feature_extraction_demo_vid.m

@@ -165,13 +165,15 @@ legend('show');
 hold off;
 
 %% Output HOG files
-[hog_data, valid_inds] = Read_HOG_file(outputHOG_aligned);
+output_hog_file = sprintf('%s/%s.hog', output_dir, name);
+[hog_data, valid_inds] = Read_HOG_file(output_hog_file);
 
 %% Output aligned images
-img_files = dir([outputDir_aligned, '/*.png']);
+output_aligned_dir = sprintf('%s/%s_aligned/', output_dir, name);
+img_files = dir([output_aligned_dir, '/*.bmp']);
 imgs = cell(numel(img_files, 1));
 for i=1:numel(img_files)
-   imgs{i} = imread([ outputDir_aligned, '/', img_files(i).name]);
+   imgs{i} = imread([ output_aligned_dir, '/', img_files(i).name]);
    imshow(imgs{i})
    drawnow
 end

matlab_runners/Demos/run_demo_align_size.m

@@ -0,0 +1,39 @@
+% A demo script that demonstrates how to process a single video file using
+% OpenFace and extract and visualize all of the features
+
+clear
+
+% The location executable will depend on the OS
+if(isunix)
+    executable = '"../../build/bin/FeatureExtraction"';
+else
+    executable = '"../../x64/Release/FeatureExtraction.exe"';
+end
+
+% Input file
+in_file = '../../samples/default.wmv';
+
+% Where to store the output
+output_dir = './processed_features/';
+
+img_sizes = [64, 112, 224];
+
+% This will take file after -f and output all the features to directory
+% after -out_dir
+command = sprintf('%s -f "%s" -out_dir "%s" -verbose -simalign', executable, in_file, output_dir);
+                 
+if(isunix)
+    unix(command);
+else
+    dos(command);
+end
+
+%% Output aligned images
+output_aligned_dir = sprintf('%s/%s_aligned/', output_dir, name);
+img_files = dir([output_aligned_dir, '/*.bmp']);
+imgs = cell(numel(img_files, 1));
+for i=1:numel(img_files)
+   imgs{i} = imread([ output_aligned_dir, '/', img_files(i).name]);
+   imshow(imgs{i})
+   drawnow
+end