diff --git a/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp index 2f04046..52fc819 100644 --- a/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp +++ b/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp @@ -44,8 +44,14 @@ RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector this->is_sequence = sequence; - this->fps_vid_out = fps_vid_out; - + if(fps_vid_out > 0) + { + this->fps_vid_out = fps_vid_out; + } + else + { + this->fps_vid_out = 30; // If an illegal value for fps provided, default to 30 + } // Default output code this->output_codec = "DIVX"; diff --git a/lib/local/Utilities/src/SequenceCapture.cpp b/lib/local/Utilities/src/SequenceCapture.cpp index cff1106..3925e1e 100644 --- a/lib/local/Utilities/src/SequenceCapture.cpp +++ b/lib/local/Utilities/src/SequenceCapture.cpp @@ -403,7 +403,6 @@ void SequenceCapture::SetCameraIntrinsics(float fx, float fy, float cx, float cy cv::Mat SequenceCapture::GetNextFrame() { - frame_num++; if (is_webcam || !is_image_seq) { @@ -429,19 +428,23 @@ cv::Mat SequenceCapture::GetNextFrame() } else if (is_image_seq) { - if (image_files.empty() || frame_num - 1 > image_files.size()) + if (image_files.empty() || frame_num >= image_files.size()) { // Indicate lack of success by returning an empty image latest_frame = cv::Mat(); } - - latest_frame = cv::imread(image_files[frame_num-1], -1); + else + { + latest_frame = cv::imread(image_files[frame_num], -1); + } time_stamp = 0; } // Set the grayscale frame convertToGrayscale(latest_frame, latest_gray_frame); + frame_num++; + return latest_frame; } diff --git a/matlab_runners/Demos/feature_extraction_demo_img_seq.m b/matlab_runners/Demos/feature_extraction_demo_img_seq.m index 5ac718c..a652645 100644 --- a/matlab_runners/Demos/feature_extraction_demo_img_seq.m +++ b/matlab_runners/Demos/feature_extraction_demo_img_seq.m @@ -1,44 +1,25 @@ +% A demo script that demonstrates how to process a single video file using +% OpenFace and extract and visualize all of the features + clear +% The location executable will depend on the OS if(isunix) executable = '"../../build/bin/FeatureExtraction"'; else executable = '"../../x64/Release/FeatureExtraction.exe"'; end -output = './output_features_seq/'; +% Input file +in_dir = '../../image_sequence'; -if(~exist(output, 'file')) - mkdir(output) -end - -in_dirs = {'../../image_sequence'}; -% some parameters -verbose = true; +% Where to store the output +output_dir = './processed_features/'; -command = executable; - -% Remove for a speedup -command = cat(2, command, ' -verbose '); - -% add all videos to single argument list (so as not to load the model anew -% for every video) -for i=1:numel(in_dirs) - - [~, name, ~] = fileparts(in_dirs{i}); - - % where to output tracking results - outputFile = [output name '.txt']; - outputDir_aligned = [output name]; - - outputHOG_aligned = [output name '.hog']; - - command = cat(2, command, ['-asvid -fdir "' in_dirs{i} '" -of "' outputFile '" ']); - - command = cat(2, command, [' -simalign "' outputDir_aligned '" -simsize 200 -hogalign "' outputHOG_aligned '"']); +% This will take directory after -fdir and output all the features to directory +% after -out_dir +command = sprintf('%s -fdir "%s" -out_dir "%s" -verbose', executable, in_dir, output_dir); -end - if(isunix) unix(command); else @@ -47,15 +28,30 @@ end %% Demonstrating reading the output files -% First read in the column names -tab = readtable(outputFile); +% Most of the features will be in the csv file in the output directory with +% the same name as the input file +[~,name,~] = fileparts(in_file); +output_csv = sprintf('%s/%s.csv', output_dir, name); + +% First read in the column names, to know which columns to read for +% particular features +tab = readtable(output_csv); column_names = tab.Properties.VariableNames; -all_params = dlmread(outputFile, ',', 1, 0); +% Read all of the data +all_params = dlmread(output_csv, ',', 1, 0); % This indicates which frames were succesfully tracked -valid_frames = logical(all_params(:,4)); -time = all_params(valid_frames, 2); + +% Find which column contains success of tracking data and timestamp data +valid_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'success')); +time_stamp_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'timestamp')); + +% Extract tracking success data and only read those frame +valid_frames = logical(all_params(:,valid_ind)); + +% Get the timestamp data +time_stamps = all_params(valid_frames, time_stamp_ind); %% Finding which header line starts with p_ (basically model params) shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_')); @@ -64,7 +60,7 @@ shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_')); shape_params = all_params(valid_frames, shape_inds); figure -plot(time, shape_params); +plot(time_stamps, shape_params); title('Shape parameters'); xlabel('Time (s)'); @@ -75,10 +71,20 @@ landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'y_')) xs = all_params(valid_frames, landmark_inds_x); ys = all_params(valid_frames, landmark_inds_y); +eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_x_')); +eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_y_')); + +eye_xs = all_params(valid_frames, eye_landmark_inds_x); +eye_ys = all_params(valid_frames, eye_landmark_inds_y); + figure for j = 1:size(xs,1) plot(xs(j,:), -ys(j,:), '.'); + hold on; + plot(eye_xs(j,:), -eye_ys(j,:), '.r'); + hold off; + xlim([min(xs(1,:)) * 0.5, max(xs(2,:))*1.4]); ylim([min(-ys(1,:)) * 1.4, max(-ys(2,:))*0.5]); xlabel('x (px)'); @@ -96,9 +102,20 @@ xs = all_params(valid_frames, landmark_inds_x); ys = all_params(valid_frames, landmark_inds_y); zs = all_params(valid_frames, landmark_inds_z); +eye_landmark_inds_x = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_X_')); +eye_landmark_inds_y = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_Y_')); +eye_landmark_inds_z = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'eye_lmk_Z_')); + +eye_xs = all_params(valid_frames, eye_landmark_inds_x); +eye_ys = all_params(valid_frames, eye_landmark_inds_y); +eye_zs = all_params(valid_frames, eye_landmark_inds_z); + figure for j = 1:size(xs,1) plot3(xs(j,:), ys(j,:), zs(j,:), '.');axis equal; + hold on; + plot3(eye_xs(j,:), eye_ys(j,:), eye_zs(j,:), '.r'); + hold off; xlabel('X (mm)'); ylabel('Y (mm)'); zlabel('Z (mm)'); @@ -110,7 +127,7 @@ au_reg_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r')); aus = all_params(valid_frames, au_reg_inds); figure -plot(aus); +plot(time, aus); title('Facial Action Units (intensity)'); xlabel('Time (s)'); ylabel('Intensity'); @@ -120,7 +137,7 @@ au_class_inds = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_c')); aus = all_params(valid_frames, au_class_inds); figure -plot(aus); +plot(time, aus); title('Facial Action Units (presense)'); xlabel('Time (s)'); ylim([0,2]); @@ -133,14 +150,30 @@ plot(pose); title('Pose (rotation and translation)'); xlabel('Time (s)'); +%% Demo gaze +gaze_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'gaze_angle')); + +% Read gaze (x,y,z) for one eye and (x,y,z) for another +gaze = all_params(valid_frames, gaze_inds); + +plot(time, gaze(:,1), 'DisplayName', 'Left - right'); +hold on; +plot(time, gaze(:,2), 'DisplayName', 'Up - down'); +xlabel('Time(s)') % x-axis label +ylabel('Angle radians') % y-axis label +legend('show'); +hold off; + %% Output HOG files -[hog_data, valid_inds] = Read_HOG_file(outputHOG_aligned); +output_hog_file = sprintf('%s/%s.hog', output_dir, name); +[hog_data, valid_inds] = Read_HOG_file(output_hog_file); %% Output aligned images -img_files = dir([outputDir_aligned, '/*.png']); +output_aligned_dir = sprintf('%s/%s_aligned/', output_dir, name); +img_files = dir([output_aligned_dir, '/*.bmp']); imgs = cell(numel(img_files, 1)); for i=1:numel(img_files) - imgs{i} = imread([ outputDir_aligned, '/', img_files(i).name]); + imgs{i} = imread([ output_aligned_dir, '/', img_files(i).name]); imshow(imgs{i}) drawnow end \ No newline at end of file diff --git a/matlab_runners/Demos/feature_extraction_demo_vid.m b/matlab_runners/Demos/feature_extraction_demo_vid.m index b56a356..aa37739 100644 --- a/matlab_runners/Demos/feature_extraction_demo_vid.m +++ b/matlab_runners/Demos/feature_extraction_demo_vid.m @@ -165,13 +165,15 @@ legend('show'); hold off; %% Output HOG files -[hog_data, valid_inds] = Read_HOG_file(outputHOG_aligned); +output_hog_file = sprintf('%s/%s.hog', output_dir, name); +[hog_data, valid_inds] = Read_HOG_file(output_hog_file); %% Output aligned images -img_files = dir([outputDir_aligned, '/*.png']); +output_aligned_dir = sprintf('%s/%s_aligned/', output_dir, name); +img_files = dir([output_aligned_dir, '/*.bmp']); imgs = cell(numel(img_files, 1)); for i=1:numel(img_files) - imgs{i} = imread([ outputDir_aligned, '/', img_files(i).name]); + imgs{i} = imread([ output_aligned_dir, '/', img_files(i).name]); imshow(imgs{i}) drawnow end \ No newline at end of file diff --git a/matlab_runners/Demos/run_demo_align_size.m b/matlab_runners/Demos/run_demo_align_size.m new file mode 100644 index 0000000..3f63011 --- /dev/null +++ b/matlab_runners/Demos/run_demo_align_size.m @@ -0,0 +1,39 @@ +% A demo script that demonstrates how to process a single video file using +% OpenFace and extract and visualize all of the features + +clear + +% The location executable will depend on the OS +if(isunix) + executable = '"../../build/bin/FeatureExtraction"'; +else + executable = '"../../x64/Release/FeatureExtraction.exe"'; +end + +% Input file +in_file = '../../samples/default.wmv'; + +% Where to store the output +output_dir = './processed_features/'; + +img_sizes = [64, 112, 224]; + +% This will take file after -f and output all the features to directory +% after -out_dir +command = sprintf('%s -f "%s" -out_dir "%s" -verbose -simalign', executable, in_file, output_dir); + +if(isunix) + unix(command); +else + dos(command); +end + +%% Output aligned images +output_aligned_dir = sprintf('%s/%s_aligned/', output_dir, name); +img_files = dir([output_aligned_dir, '/*.bmp']); +imgs = cell(numel(img_files, 1)); +for i=1:numel(img_files) + imgs{i} = imread([ output_aligned_dir, '/', img_files(i).name]); + imshow(imgs{i}) + drawnow +end \ No newline at end of file