r/sustaining_gazes
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

More work at establishing AU baselines on SEMAINE and BP4D datasets.

Browse source
This commit is contained in:
Tadas Baltrusaitis 2016-05-24 16:51:18 -04:00
parent a103d6558a
commit 803575d5b7
18 changed files with 968 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitignore vendored
View file

@ -20,3 +20,9 @@ matlab_runners/Feature Point Experiments/yt_features/
matlab_runners/Feature Point Experiments/yt_features_clm/
matlab_runners/Gaze Experiments/mpii_out/
build/
Release/
matlab_runners/Action Unit Experiments/out_bp4d/
matlab_runners/Action Unit Experiments/out_SEMAINE/
*.ipch
exe/FeatureExtraction/out_bp4d/
x64/Debug/

4
exe/FeatureExtraction/FeatureExtraction.cpp
View file

@ -690,6 +690,8 @@ int main (int argc, char **argv)
// If the video is long enough post-process it for AUs
if (output_AUs && frame_count > 100)
{
cout << "Postprocessing the Action Unit predictions" << endl;
post_process_output_file(face_analyser, output_files[f_n]);
}
}
@ -706,7 +708,7 @@ int main (int argc, char **argv)
}
// break out of the loop if done with all the files (or using a webcam)
if(f_n == input_files.size() -1 || input_files.empty())
if((video_input && f_n == input_files.size() -1) || (!video_input && f_n == input_image_files.size() - 1))
{
done = true;
}

11
lib/local/FaceAnalyser/include/FaceAnalyser.h
View file

@ -212,7 +212,7 @@ private:
void UpdatePredictionTrack(cv::Mat_<unsigned int>& prediction_corr_histogram, int& prediction_correction_count, vector<double>& correction, const vector<pair<string, double>>& predictions, double ratio=0.25, int num_bins = 200, double min_val = -3, double max_val = 5, int min_frames = 10);
void GetSampleHist(cv::Mat_<unsigned int>& prediction_corr_histogram, int prediction_correction_count, vector<double>& sample, double ratio, int num_bins = 200, double min_val = 0, double max_val = 5);
vector<std::pair<std::string, vector<double>>> PostprocessPredictions();
void PostprocessPredictions();
vector<cv::Mat_<unsigned int>> au_prediction_correction_histogram;
vector<int> au_prediction_correction_count;
@ -233,6 +233,15 @@ private:
double align_scale;
int align_width;
int align_height;
// Useful placeholder for renormalizing the initial frames of shorter videos
int max_init_frames = 3000;
vector<cv::Mat_<double>> hog_desc_frames_init;
vector<cv::Mat_<double>> geom_descriptor_frames_init;
vector<int> views;
bool postprocessed = false;
int frames_tracking_succ = 0;
};
//===========================================================================
}

96
lib/local/FaceAnalyser/src/FaceAnalyser.cpp
View file

@ -321,6 +321,9 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL
update_median = update_median & clnf_model.detection_success;
if (clnf_model.detection_success)
frames_tracking_succ++;
// A small speedup
if(frames_tracking % 2 == 1)
{
@ -368,16 +371,13 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL
AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success);
}
// Keep only closer to in-plane faces
double angle_norm = cv::sqrt(clnf_model.params_global[2] * clnf_model.params_global[2] + clnf_model.params_global[3] * clnf_model.params_global[3]);
// Add the reg predictions to the historic data
for (size_t au = 0; au < AU_predictions_reg.size(); ++au)
{
// Find the appropriate AU (if not found add it)
// Only add if the detection was successful and not too out of plane
if(clnf_model.detection_success && angle_norm < 0.4)
// Only add if the detection was successful
if(clnf_model.detection_success)
{
AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(AU_predictions_reg[au].second);
}
@ -393,8 +393,8 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL
{
// Find the appropriate AU (if not found add it)
// Only add if the detection was successful and not too out of plane
if(clnf_model.detection_success && angle_norm < 0.4)
// Only add if the detection was successful
if(clnf_model.detection_success)
{
AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(AU_predictions_class[au].second);
}
@ -409,16 +409,28 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL
{
AU_predictions_reg = AU_predictions_reg_corrected;
}
else
{
if (clnf_model.detection_success && frames_tracking_succ - 1 < max_init_frames)
{
hog_desc_frames_init.push_back(hog_descriptor);
geom_descriptor_frames_init.push_back(geom_descriptor_frame);
views.push_back(orientation_to_use);
}
}
this->current_time_seconds = timestamp_seconds;
view_used = orientation_to_use;
bool success = clnf_model.detection_success && angle_norm < 0.4;
bool success = clnf_model.detection_success;
confidences.push_back(clnf_model.detection_certainty);
valid_preds.push_back(success);
timestamps.push_back(timestamp_seconds);
}
void FaceAnalyser::GetGeomDescriptor(cv::Mat_<double>& geom_desc)
@ -444,16 +456,13 @@ void FaceAnalyser::PredictAUs(const cv::Mat_<double>& hog_features, const cv::Ma
AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success);
}
// Keep only closer to in-plane faces
double angle_norm = cv::sqrt(clnf_model.params_global[2] * clnf_model.params_global[2] + clnf_model.params_global[3] * clnf_model.params_global[3]);
// Add the reg predictions to the historic data
for (size_t au = 0; au < AU_predictions_reg.size(); ++au)
{
// Find the appropriate AU (if not found add it)
// Only add if the detection was successful and not too out of plane
if(clnf_model.detection_success && angle_norm < 0.4)
// Only add if the detection was successful
if(clnf_model.detection_success)
{
AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(AU_predictions_reg[au].second);
}
@ -469,8 +478,8 @@ void FaceAnalyser::PredictAUs(const cv::Mat_<double>& hog_features, const cv::Ma
{
// Find the appropriate AU (if not found add it)
// Only add if the detection was successful and not too out of plane
if(clnf_model.detection_success && angle_norm < 0.4)
// Only add if the detection was successful
if(clnf_model.detection_success)
{
AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(AU_predictions_class[au].second);
}
@ -496,14 +505,62 @@ void FaceAnalyser::PredictAUs(const cv::Mat_<double>& hog_features, const cv::Ma
view_used = orientation_to_use;
bool success = clnf_model.detection_success && angle_norm < 0.4;
bool success = clnf_model.detection_success;
confidences.push_back(clnf_model.detection_certainty);
valid_preds.push_back(success);
}
// Perform prediction on initial n frames anew as the current neutral face estimate is better now
void FaceAnalyser::PostprocessPredictions()
{
if(!postprocessed)
{
int success_ind = 0;
int all_ind = 0;
int all_frames_size = timestamps.size();
while(all_ind < all_frames_size && success_ind < max_init_frames)
{
if(valid_preds[all_ind])
{
this->hog_desc_frame = hog_desc_frames_init[success_ind];
this->geom_descriptor_frame = geom_descriptor_frames_init[success_ind];
// Perform AU prediction
auto AU_predictions_reg = PredictCurrentAUs(views[success_ind]);
// Modify the predictions to the historic data
for (size_t au = 0; au < AU_predictions_reg.size(); ++au)
{
// Find the appropriate AU (if not found add it)
AU_predictions_reg_all_hist[AU_predictions_reg[au].first][all_ind] = AU_predictions_reg[au].second;
}
auto AU_predictions_class = PredictCurrentAUsClass(views[success_ind]);
for (size_t au = 0; au < AU_predictions_class.size(); ++au)
{
// Find the appropriate AU (if not found add it)
AU_predictions_class_all_hist[AU_predictions_class[au].first][all_ind] = AU_predictions_class[au].second;
}
success_ind++;
}
all_ind++;
}
postprocessed = true;
}
}
void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps)
{
PostprocessPredictions();
timestamps = this->timestamps;
au_predictions.clear();
// First extract the valid AU values and put them in a different format
@ -573,6 +630,8 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector<std::pair<std::string,
void FaceAnalyser::ExtractAllPredictionsOfflineClass(vector<std::pair<std::string, vector<double>>>& au_predictions, vector<double>& confidences, vector<bool>& successes, vector<double>& timestamps)
{
PostprocessPredictions();
timestamps = this->timestamps;
au_predictions.clear();
@ -631,6 +690,11 @@ void FaceAnalyser::Reset()
confidences.clear();
valid_preds.clear();
// Clean up the postprocessing data as well
hog_desc_frames_init.clear();
geom_descriptor_frames_init.clear();
postprocessed = false;
frames_tracking_succ = 0;
}
void FaceAnalyser::UpdateRunningMedian(cv::Mat_<unsigned int>& histogram, int& hist_count, cv::Mat_<double>& median, const cv::Mat_<double>& descriptor, bool update, int num_bins, double min_val, double max_val)

6
matlab_runners/Action Unit Experiments/BP4D_valid_res_class.txt Normal file
View file

@ -0,0 +1,6 @@
AU6 intensity, Precision - 0.182, Recall - 0.050, F1 - 0.079
AU10 intensity, Precision - 0.200, Recall - 0.058, F1 - 0.090
AU12 intensity, Precision - 0.361, Recall - 0.081, F1 - 0.132
AU12 class, Precision - 0.494, Recall - 0.551, F1 - 0.521
AU14 intensity, Precision - 0.098, Recall - 0.059, F1 - 0.074
AU17 intensity, Precision - 0.358, Recall - 0.048, F1 - 0.085

5
matlab_runners/Action Unit Experiments/BP4D_valid_res_int.txt Normal file
View file

@ -0,0 +1,5 @@
AU6 results - corr 0.548, ccc - 0.319
AU10 results - corr 0.415, ccc - 0.216
AU12 results - corr 0.495, ccc - 0.297
AU14 results - corr 0.241, ccc - 0.122
AU17 results - corr 0.456, ccc - 0.253

22
matlab_runners/Action Unit Experiments/DISFA_valid_res.txt
View file

@ -1,12 +1,12 @@
AU1 results - corr 0.740, ccc - 0.729
AU2 results - corr 0.702, ccc - 0.623
AU4 results - corr 0.847, ccc - 0.823
AU5 results - corr 0.669, ccc - 0.662
AU6 results - corr 0.641, ccc - 0.627
AU9 results - corr 0.722, ccc - 0.703
AU12 results - corr 0.857, ccc - 0.847
AU15 results - corr 0.652, ccc - 0.639
AU1 results - corr 0.735, ccc - 0.724
AU2 results - corr 0.684, ccc - 0.624
AU4 results - corr 0.836, ccc - 0.815
AU5 results - corr 0.635, ccc - 0.622
AU6 results - corr 0.641, ccc - 0.626
AU9 results - corr 0.703, ccc - 0.681
AU12 results - corr 0.861, ccc - 0.850
AU15 results - corr 0.630, ccc - 0.620
AU17 results - corr 0.572, ccc - 0.506
AU20 results - corr 0.507, ccc - 0.489
AU25 results - corr 0.918, ccc - 0.916
AU26 results - corr 0.552, ccc - 0.185
AU20 results - corr 0.486, ccc - 0.450
AU25 results - corr 0.920, ccc - 0.918
AU26 results - corr 0.586, ccc - 0.220

7
matlab_runners/Action Unit Experiments/SEMAINE_valid_res.txt Normal file
View file

@ -0,0 +1,7 @@
AU2 intensity, Precision - 0.454, Recall - 0.462, F1 - 0.458
AU12 intensity, Precision - 0.510, Recall - 0.393, F1 - 0.444
AU12 class, Precision - 0.406, Recall - 0.828, F1 - 0.545
AU17 intensity, Precision - 0.394, Recall - 0.392, F1 - 0.393
AU25 intensity, Precision - 0.342, Recall - 0.768, F1 - 0.473
AU28 class, Precision - 0.613, Recall - 0.500, F1 - 0.551
AU45 class, Precision - 0.300, Recall - 0.636, F1 - 0.408

67
matlab_runners/Action Unit Experiments/helpers/extract_BP4D_labels.m Normal file
View file

@ -0,0 +1,67 @@
function [ labels, valid_ids, vid_ids, filenames ] = extract_BP4D_labels( BP4D_dir, recs, aus )
%EXTRACT_SEMAINE_LABELS Summary of this function goes here
% Detailed explanation goes here
aus_BP4D = [1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23];
inds_to_use = [];
for i=1:numel(aus)
inds_to_use = cat(1, inds_to_use, find(aus_BP4D == aus(i)));
end
num_files = numel(dir([BP4D_dir, '/*.csv']));
labels = cell(num_files, 1);
valid_ids = cell(num_files, 1);
vid_ids = zeros(num_files, 2);
filenames = cell(num_files, 1);
file_id = 1;
for i=1:numel(recs)
csvs = dir([BP4D_dir, '/', recs{i}, '*.csv']);
for f=1:numel(csvs)
file = [BP4D_dir, '/', csvs(f).name];
[~, filename,~] = fileparts(file);
filenames{file_id} = filename;
OCC = csvread(file); %import annotations for one video file
frame_nums = OCC(2:end,1); %get all frame numbers
codes = OCC(2:end,2:end); %get codes for all action units
occlusions = OCC(2:end,end);
codes = codes(:, aus_BP4D);
% Finding the invalid regions
valid = occlusions ~= 1;
for s=1:size(codes,2)
valid = valid & codes(:,s) ~= 9;
end
vid_ids(file_id,:) = [frame_nums(1), frame_nums(end)];
labels{file_id} = codes(:, inds_to_use);
% all indices in SEMAINE are valid
valid_ids{file_id} = valid;
file_id = file_id + 1;
end
end
labels = labels(1:file_id-1);
valid_ids = valid_ids(1:file_id-1);
vid_ids = vid_ids(1:file_id-1, :);
filenames = filenames(1:file_id-1);
end

63
matlab_runners/Action Unit Experiments/helpers/extract_BP4D_labels_intensity.m Normal file
View file

@ -0,0 +1,63 @@
function [ labels, valid_ids, vid_ids, filenames ] = extract_BP4D_labels_intensity( BP4D_dir, recs, aus )
%EXTRACT_SEMAINE_LABELS Summary of this function goes here
% Detailed explanation goes here
files_all = dir(sprintf('%s/AU%02d/%s', BP4D_dir, aus(1), '/*.csv'));
num_files = numel(files_all);
labels = cell(num_files, 1);
valid_ids = cell(num_files, 1);
vid_ids = zeros(num_files, 2);
filenames = cell(num_files, 1);
file_id = 1;
for r=1:numel(recs)
files_root = sprintf('%s/AU%02d/', BP4D_dir, aus(1));
files_all = dir([files_root, recs{r}, '*.csv']);
for f=1:numel(files_all)
for au=aus
% Need to find relevant files for the relevant user and for the
% relevant AU
files_root = sprintf('%s/AU%02d/', BP4D_dir, au);
files_all = dir([files_root, recs{r}, '*.csv']);
file = [files_root, '/', files_all(f).name];
[~, filename,~] = fileparts(file);
filenames{file_id} = filename(1:7);
intensities = csvread(file); % import annotations for one session
frame_nums = intensities(:,1); % get all frame numbers
codes = intensities(:,2);
% Finding the invalid regions
valid = codes ~= 9;
vid_ids(file_id,:) = [frame_nums(1), frame_nums(end)];
if(au == aus(1))
valid_ids{file_id} = valid;
labels{file_id} = codes;
else
valid_ids{file_id} = valid_ids{file_id} & valid;
labels{file_id} = cat(2, labels{file_id}, codes);
end
end
file_id = file_id + 1;
end
end
labels = labels(1:file_id-1);
valid_ids = valid_ids(1:file_id-1);
vid_ids = vid_ids(1:file_id-1, :);
filenames = filenames(1:file_id-1);
end

52
matlab_runners/Action Unit Experiments/helpers/extract_SEMAINE_labels.m Normal file
View file

@ -0,0 +1,52 @@
function [ labels, valid_ids, vid_ids ] = extract_SEMAINE_labels( SEMAINE_dir, recs, aus )
%EXTRACT_SEMAINE_LABELS Summary of this function goes here
% Detailed explanation goes here
% Get the right eaf file
aus_SEMAINE = [2 12 17 25 28 45];
inds_to_use = [];
for i=1:numel(aus)
inds_to_use = cat(1, inds_to_use, find(aus_SEMAINE == aus(i)));
end
labels = cell(numel(recs), 1);
valid_ids = cell(numel(recs), 1);
vid_ids = zeros(numel(recs), 2);
for i=1:numel(recs)
file = dir([SEMAINE_dir, '/', recs{i}, '/*.eaf']);
vid_ids(i,:) = dlmread([SEMAINE_dir, '/', recs{i}, '.txt'], ' ');
xml_file = [SEMAINE_dir, recs{i}, '\' file.name];
[root_xml, name_xml, ~] = fileparts(xml_file);
m_file = [root_xml, name_xml, '.mat'];
if(~exist(m_file, 'file'))
activations = ParseSEMAINEAnnotations([SEMAINE_dir, recs{i}, '\' file.name]);
save(m_file, 'activations');
else
load(m_file);
end
if(size(activations,1) < vid_ids(i,2))
vid_ids(i,2) = size(activations,1);
if(vid_ids(i,2) > 2999)
vid_ids(i,1) = vid_ids(i,2) - 2999;
end
end
labels{i} = activations(vid_ids(i,1)+1:vid_ids(i,2), 1 + inds_to_use);
% all indices in SEMAINE are valid
valid_ids{i} = ones(size(labels{i},1),1);
end
end

28
matlab_runners/Action Unit Experiments/helpers/find_BP4D.m Normal file
View file

@ -0,0 +1,28 @@
if(exist('C:\tadas\face_datasets\fera_2015\bp4d\AUCoding/', 'file'))
BP4D_dir = 'C:\tadas\face_datasets\fera_2015\bp4d\AUCoding/';
BP4D_dir_int = 'C:\tadas\face_datasets\fera_2015\bp4d\AU Intensity Codes3.0/';
elseif(exist('E:\datasets\FERA_2015\BP4D\AUCoding/', 'file'))
BP4D_dir = 'E:\datasets\FERA_2015\BP4D\AUCoding/';
BP4D_dir_int = 'E:\datasets\FERA_2015\BP4D\AU Intensity Codes3.0/';
elseif(exist('D:\datasets\face_datasets\fera_2015\bp4d\AUCoding/','file'))
BP4D_dir = 'D:\datasets\face_datasets\fera_2015\bp4d\AUCoding/';
BP4D_dir_int = 'D:\datasets\face_datasets\fera_2015\bp4d\AU Intensity Codes3.0/';
elseif(exist('D:\Datasets\FERA_2015\BP4D\AUCoding/','file'))
BP4D_dir = 'D:\Datasets\FERA_2015\BP4D\AUCoding/';
BP4D_dir_int = 'D:\Datasets\FERA_2015\BP4D\AU Intensity Codes3.0/';
elseif(exist('I:\datasets\FERA_2015\BP4D\AUCoding/', 'file'))
BP4D_dir = 'I:\datasets\FERA_2015\BP4D\AUCoding/';
BP4D_dir_int = 'I:\datasets\FERA_2015\BP4D\AU Intensity Codes3.0/';
elseif(exist('D:/fera_2015/bp4d/AUCoding/', 'file'))
BP4D_dir = 'D:/fera_2015/bp4d/AUCoding/';
BP4D_dir_int = 'D:/fera_2015/bp4d/AU Intensity Codes3.0/';
else
fprintf('BP4D location not found (or not defined)\n');
end
hog_data_dir = [BP4D_dir, '../processed_data'];
train_recs = {'F001', 'F003', 'F005', 'F007', 'F009', 'F011', 'F013', 'F015', 'F017', 'F019', 'F021', 'F023', 'M001', 'M003', 'M005', 'M007', 'M009', 'M011', 'M013', 'M015' 'M017'};
devel_recs = {'F002', 'F004', 'F006', 'F008', 'F010', 'F012', 'F014', 'F016', 'F018', 'F020', 'F022', 'M002', 'M004', 'M006', 'M008', 'M010', 'M012', 'M014', 'M016', 'M018'};

24
matlab_runners/Action Unit Experiments/helpers/find_SEMAINE.m Normal file
View file

@ -0,0 +1,24 @@
if(exist('E:\datasets\FERA_2015\semaine/SEMAINE-Sessions/', 'file'))
SEMAINE_dir = 'E:\datasets\FERA_2015\semaine/SEMAINE-Sessions/';
elseif(exist('I:\datasets\FERA_2015\Semaine\SEMAINE-Sessions/', 'file'))
SEMAINE_dir = 'I:\datasets\FERA_2015\Semaine\SEMAINE-Sessions/';
elseif(exist('C:\tadas\face_datasets\fera_2015\semaine/SEMAINE-Sessions/', 'file'))
SEMAINE_dir = 'C:\tadas\face_datasets\fera_2015\semaine/SEMAINE-Sessions/';
elseif(exist('D:\datasets\face_datasets\fera_2015\semaine\SEMAINE-Sessions/', 'file'))
SEMAINE_dir = 'D:\datasets\face_datasets\fera_2015\semaine\SEMAINE-Sessions/';
elseif(exist('D:\Datasets\FERA_2015\semaine\SEMAINE-Sessions/', 'file'))
SEMAINE_dir = 'D:\Datasets\FERA_2015\semaine\SEMAINE-Sessions/';
elseif(exist('D:/fera_2015/semaine/SEMAINE-Sessions/', 'file'))
SEMAINE_dir = 'D:/fera_2015/semaine/SEMAINE-Sessions/';
else
fprintf('DISFA location not found (or not defined)\n');
end
if(exist('SEMAINE_dir', 'var'))
hog_data_dir = [SEMAINE_dir, '../processed_data/'];
end
train_recs = {'rec1', 'rec12', 'rec14', 'rec19', 'rec23', 'rec25', 'rec37', 'rec39', 'rec43', 'rec45', 'rec48', 'rec50', 'rec52', 'rec54', 'rec56', 'rec60'};
devel_recs = {'rec9', 'rec13', 'rec15', 'rec20', 'rec24', 'rec26', 'rec38', 'rec42', 'rec44', 'rec46', 'rec49', 'rec51', 'rec53', 'rec55', 'rec58'};
aus_SEMAINE = [2 12 17 25 28 45];

226
matlab_runners/Action Unit Experiments/run_AU_prediction_BP4D.asv Normal file
View file

@ -0,0 +1,226 @@
clear
bp4d_loc = 'D:/Datasets/FERA_2015/BP4D/BP4D-training/';
out_loc = './out_bp4d/';
if(~exist(out_loc, 'dir'))
mkdir(out_loc);
end
%%
executable = '"../../x64/Release/FeatureExtraction.exe"';
bp4d_dirs = {'F002', 'F004', 'F006', 'F008', 'F010', 'F012', 'F014', 'F016', 'F018', 'F020', 'F022', 'M002', 'M004', 'M006', 'M008', 'M010', 'M012', 'M014', 'M016', 'M018'};
parfor f1=1:numel(bp4d_dirs)
if(isdir([bp4d_loc, bp4d_dirs{f1}]))
bp4d_2_dirs = dir([bp4d_loc, bp4d_dirs{f1}]);
bp4d_2_dirs = bp4d_2_dirs(3:end);
f1_dir = bp4d_dirs{f1};
command = [executable ' -asvid -q -no2Dfp -no3Dfp -noMparams -noPose -noGaze '];
for f2=1:numel(bp4d_2_dirs)
f2_dir = bp4d_2_dirs(f2).name;
if(isdir([bp4d_loc, bp4d_dirs{f1}]))
curr_vid = [bp4d_loc, f1_dir, '/', f2_dir, '/'];
name = [f1_dir '_' f2_dir];
output_file = [out_loc name '.au.txt'];
command = cat(2, command, [' -fdir "' curr_vid '" -of "' output_file '"']);
end
end
dos(command);
end
end
%%
addpath('./helpers/');
find_BP4D;
aus_BP4D = [1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23];
[ labels_gt, valid_ids, vid_ids, filenames] = extract_BP4D_labels(BP4D_dir, bp4d_dirs, aus_BP4D);
labels_gt = cat(1, labels_gt{:});
%% Identifying which column IDs correspond to which AU
tab = readtable([out_loc, bp4d_dirs{1}, '_T1.au.txt']);
column_names = tab.Properties.VariableNames;
% As there are both classes and intensities list and evaluate both of them
aus_pred_int = [];
aus_pred_class = [];
inds_int_in_file = [];
inds_class_in_file = [];
for c=1:numel(column_names)
if(strfind(column_names{c}, '_r') > 0)
aus_pred_int = cat(1, aus_pred_int, int32(str2num(column_names{c}(3:end-2))));
inds_int_in_file = cat(1, inds_int_in_file, c);
end
if(strfind(column_names{c}, '_c') > 0)
aus_pred_class = cat(1, aus_pred_class, int32(str2num(column_names{c}(3:end-2))));
inds_class_in_file = cat(1, inds_class_in_file, c);
end
end
%%
inds_au_int = zeros(size(aus_BP4D));
inds_au_class = zeros(size(aus_BP4D));
for ind=1:numel(aus_BP4D)
if(~isempty(find(aus_pred_int==aus_BP4D(ind), 1)))
inds_au_int(ind) = find(aus_pred_int==aus_BP4D(ind));
end
end
for ind=1:numel(aus_BP4D)
if(~isempty(find(aus_pred_class==aus_BP4D(ind), 1)))
inds_au_class(ind) = find(aus_pred_class==aus_BP4D(ind));
end
end
preds_all_class = [];
preds_all_int = [];
for i=1:numel(filenames)
fname = [out_loc, filenames{i}, '.au.txt'];
preds = dlmread(fname, ',', 1, 0);
% Read all of the intensity AUs
preds_int = preds(:, inds_int_in_file);
% Read all of the classification AUs
preds_class = preds(:, inds_class_in_file);
preds_all_class = cat(1, preds_all_class, preds_class);
preds_all_int = cat(1, preds_all_int, preds_int);
end
%%
f = fopen('BP4D_valid_res_class.txt', 'w');
for au = 1:numel(aus_BP4D)
if(inds_au_int(au) ~= 0)
tp = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) >= 1);
fp = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) >= 1);
fn = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) < 1);
tn = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) < 1);
precision = tp./(tp+fp);
recall = tp./(tp+fn);
f1 = 2 * precision .* recall ./ (precision + recall);
fprintf(f, 'AU%d intensity, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_BP4D(au), precision, recall, f1);
end
if(inds_au_class(au) ~= 0)
tp = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 1);
fp = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 1);
fn = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 0);
tn = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 0);
precision = tp./(tp+fp);
recall = tp./(tp+fn);
f1 = 2 * precision .* recall ./ (precision + recall);
fprintf(f, 'AU%d class, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_BP4D(au), precision, recall, f1);
end
end
fclose(f);
%%
addpath('./helpers/');
find_BP4D;
aus_BP4D = [6, 10, 12, 14, 17];
[ labels_gt, valid_ids, vid_ids, filenames] = extract_BP4D_labels_intensity(BP4D_dir_int, devel_recs, aus_BP4D);
labels_gt = cat(1, labels_gt{:});
%% Identifying which column IDs correspond to which AU
tab = readtable([out_loc, bp4d_dirs{1}, '_T1.au.txt']);
column_names = tab.Properties.VariableNames;
% As there are both classes and intensities list and evaluate both of them
aus_pred_int = [];
inds_int_in_file = [];
for c=1:numel(column_names)
if(strfind(column_names{c}, '_r') > 0)
aus_pred_int = cat(1, aus_pred_int, int32(str2num(column_names{c}(3:end-2))));
inds_int_in_file = cat(1, inds_int_in_file, c);
end
end
%%
inds_au_int = zeros(size(aus_BP4D));
for ind=1:numel(aus_BP4D)
if(~isempty(find(aus_pred_int==aus_BP4D(ind), 1)))
inds_au_int(ind) = find(aus_pred_int==aus_BP4D(ind));
end
end
preds_all_class = [];
for i=1:numel(filenames)
fname = [out_loc, filenames{i}, '.au.txt'];
preds = dlmread(fname, ',', 1, 0);
% Read all of the intensity AUs
preds_int = preds(:, inds_int_in_file);
preds_all_class = cat(1, preds_all_class, preds_class);
preds_all_int = cat(1, preds_all_int, preds_int);
end
%%
f = fopen('BP4D_valid_res_class.txt', 'w');
for au = 1:numel(aus_BP4D)
if(inds_au_int(au) ~= 0)
tp = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) >= 1);
fp = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) >= 1);
fn = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) < 1);
tn = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) < 1);
precision = tp./(tp+fp);
recall = tp./(tp+fn);
f1 = 2 * precision .* recall ./ (precision + recall);
fprintf(f, 'AU%d intensity, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_BP4D(au), precision, recall, f1);
end
if(inds_au_class(au) ~= 0)
tp = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 1);
fp = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 1);
fn = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 0);
tn = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 0);
precision = tp./(tp+fp);
recall = tp./(tp+fn);
f1 = 2 * precision .* recall ./ (precision + recall);
fprintf(f, 'AU%d class, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_BP4D(au), precision, recall, f1);
end
end
fclose(f);

196
matlab_runners/Action Unit Experiments/run_AU_prediction_BP4D.m Normal file
View file

@ -0,0 +1,196 @@
clear
bp4d_loc = 'D:/Datasets/FERA_2015/BP4D/BP4D-training/';
out_loc = './out_bp4d/';
if(~exist(out_loc, 'dir'))
mkdir(out_loc);
end
%%
executable = '"../../x64/Release/FeatureExtraction.exe"';
bp4d_dirs = {'F002', 'F004', 'F006', 'F008', 'F010', 'F012', 'F014', 'F016', 'F018', 'F020', 'F022', 'M002', 'M004', 'M006', 'M008', 'M010', 'M012', 'M014', 'M016', 'M018'};
parfor f1=1:numel(bp4d_dirs)
if(isdir([bp4d_loc, bp4d_dirs{f1}]))
bp4d_2_dirs = dir([bp4d_loc, bp4d_dirs{f1}]);
bp4d_2_dirs = bp4d_2_dirs(3:end);
f1_dir = bp4d_dirs{f1};
command = [executable ' -asvid -q -no2Dfp -no3Dfp -noMparams -noPose -noGaze '];
for f2=1:numel(bp4d_2_dirs)
f2_dir = bp4d_2_dirs(f2).name;
if(isdir([bp4d_loc, bp4d_dirs{f1}]))
curr_vid = [bp4d_loc, f1_dir, '/', f2_dir, '/'];
name = [f1_dir '_' f2_dir];
output_file = [out_loc name '.au.txt'];
command = cat(2, command, [' -fdir "' curr_vid '" -of "' output_file '"']);
end
end
dos(command);
end
end
%%
addpath('./helpers/');
find_BP4D;
aus_BP4D = [1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23];
[ labels_gt, valid_ids, vid_ids, filenames] = extract_BP4D_labels(BP4D_dir, bp4d_dirs, aus_BP4D);
labels_gt = cat(1, labels_gt{:});
%% Identifying which column IDs correspond to which AU
tab = readtable([out_loc, bp4d_dirs{1}, '_T1.au.txt']);
column_names = tab.Properties.VariableNames;
% As there are both classes and intensities list and evaluate both of them
aus_pred_int = [];
aus_pred_class = [];
inds_int_in_file = [];
inds_class_in_file = [];
for c=1:numel(column_names)
if(strfind(column_names{c}, '_r') > 0)
aus_pred_int = cat(1, aus_pred_int, int32(str2num(column_names{c}(3:end-2))));
inds_int_in_file = cat(1, inds_int_in_file, c);
end
if(strfind(column_names{c}, '_c') > 0)
aus_pred_class = cat(1, aus_pred_class, int32(str2num(column_names{c}(3:end-2))));
inds_class_in_file = cat(1, inds_class_in_file, c);
end
end
%%
inds_au_int = zeros(size(aus_BP4D));
inds_au_class = zeros(size(aus_BP4D));
for ind=1:numel(aus_BP4D)
if(~isempty(find(aus_pred_int==aus_BP4D(ind), 1)))
inds_au_int(ind) = find(aus_pred_int==aus_BP4D(ind));
end
end
for ind=1:numel(aus_BP4D)
if(~isempty(find(aus_pred_class==aus_BP4D(ind), 1)))
inds_au_class(ind) = find(aus_pred_class==aus_BP4D(ind));
end
end
preds_all_class = [];
preds_all_int = [];
for i=1:numel(filenames)
fname = [out_loc, filenames{i}, '.au.txt'];
preds = dlmread(fname, ',', 1, 0);
% Read all of the intensity AUs
preds_int = preds(:, inds_int_in_file);
% Read all of the classification AUs
preds_class = preds(:, inds_class_in_file);
preds_all_class = cat(1, preds_all_class, preds_class);
preds_all_int = cat(1, preds_all_int, preds_int);
end
%%
f = fopen('BP4D_valid_res_class.txt', 'w');
for au = 1:numel(aus_BP4D)
if(inds_au_int(au) ~= 0)
tp = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) >= 1);
fp = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) >= 1);
fn = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) < 1);
tn = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) < 1);
precision = tp./(tp+fp);
recall = tp./(tp+fn);
f1 = 2 * precision .* recall ./ (precision + recall);
fprintf(f, 'AU%d intensity, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_BP4D(au), precision, recall, f1);
end
if(inds_au_class(au) ~= 0)
tp = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 1);
fp = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 1);
fn = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 0);
tn = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 0);
precision = tp./(tp+fp);
recall = tp./(tp+fn);
f1 = 2 * precision .* recall ./ (precision + recall);
fprintf(f, 'AU%d class, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_BP4D(au), precision, recall, f1);
end
end
fclose(f);
%%
addpath('./helpers/');
find_BP4D;
aus_BP4D = [6, 10, 12, 14, 17];
[ labels_gt, valid_ids, vid_ids, filenames] = extract_BP4D_labels_intensity(BP4D_dir_int, devel_recs, aus_BP4D);
labels_gt = cat(1, labels_gt{:});
%% Identifying which column IDs correspond to which AU
tab = readtable([out_loc, bp4d_dirs{1}, '_T1.au.txt']);
column_names = tab.Properties.VariableNames;
% As there are both classes and intensities list and evaluate both of them
aus_pred_int = [];
inds_int_in_file = [];
for c=1:numel(column_names)
if(strfind(column_names{c}, '_r') > 0)
aus_pred_int = cat(1, aus_pred_int, int32(str2num(column_names{c}(3:end-2))));
inds_int_in_file = cat(1, inds_int_in_file, c);
end
end
%%
inds_au_int = zeros(size(aus_BP4D));
for ind=1:numel(aus_BP4D)
if(~isempty(find(aus_pred_int==aus_BP4D(ind), 1)))
inds_au_int(ind) = find(aus_pred_int==aus_BP4D(ind));
end
end
preds_all_int = [];
for i=1:numel(filenames)
fname = [out_loc, filenames{i}, '.au.txt'];
preds = dlmread(fname, ',', 1, 0);
% Read all of the intensity AUs
preds_int = preds(:, inds_int_in_file);
preds_all_int = cat(1, preds_all_int, preds_int);
end
%%
f = fopen('BP4D_valid_res_int.txt', 'w');
for au = 1:numel(aus_BP4D)
[ accuracies, F1s, corrs, ccc, rms, classes ] = evaluate_au_prediction_results( preds_all_int(:, inds_au_int(au)), labels_gt(:,au));
fprintf(f, 'AU%d results - corr %.3f, ccc - %.3f\n', aus_BP4D(au), corrs, ccc);
end
fclose(f);

4
matlab_runners/Action Unit Experiments/run_AU_prediction_DISFA.m
View file

@ -1,6 +1,6 @@
clear
DISFA_dir = 'D:/Datasets/DISFA/Videos_LeftCamera/';
clm_exe = '"../../x64/Release/FeatureExtraction.exe"';
executable = '"../../x64/Release/FeatureExtraction.exe"';
videos = dir([DISFA_dir, '*.avi']);
@ -20,7 +20,7 @@ parfor v = 1:numel(videos)
% where to output tracking results
output_file = [output name '_au.txt'];
command = [clm_exe ' -f "' vid_file '" -of "' output_file '" -q -no2Dfp -no3Dfp -noMparams -noPose -noGaze'];
command = [executable ' -f "' vid_file '" -of "' output_file '" -q -no2Dfp -no3Dfp -noMparams -noPose -noGaze'];
dos(command);

130
matlab_runners/Action Unit Experiments/run_AU_prediction_SEMAINE.m Normal file
View file

@ -0,0 +1,130 @@
clear
addpath(genpath('helpers/'));
find_SEMAINE;
out_loc = './out_SEMAINE/';
if(~exist(out_loc, 'dir'))
mkdir(out_loc);
end
executable = '"../../x64/Release/FeatureExtraction.exe"';
parfor f1=1:numel(devel_recs)
if(isdir([SEMAINE_dir, devel_recs{f1}]))
vid_file = dir([SEMAINE_dir, devel_recs{f1}, '/*.avi']);
f1_dir = devel_recs{f1};
command = [executable, ' -fx 800 -fy 800 -rigid -q -no2Dfp -no3Dfp -noMparams -noPose -noGaze '];
curr_vid = [SEMAINE_dir, f1_dir, '/', vid_file.name];
name = f1_dir;
output_aus = [out_loc name '.au.txt'];
command = cat(2, command, [' -f "' curr_vid '" -of "' output_aus]);
dos(command);
end
end
%% Actual model evaluation
[ labels, valid_ids, vid_ids ] = extract_SEMAINE_labels(SEMAINE_dir, devel_recs, aus_SEMAINE);
labels_gt = cat(1, labels{:});
%% Identifying which column IDs correspond to which AU
tab = readtable([out_loc, devel_recs{1}, '.au.txt']);
column_names = tab.Properties.VariableNames;
% As there are both classes and intensities list and evaluate both of them
aus_pred_int = [];
aus_pred_class = [];
inds_int_in_file = [];
inds_class_in_file = [];
for c=1:numel(column_names)
if(strfind(column_names{c}, '_r') > 0)
aus_pred_int = cat(1, aus_pred_int, int32(str2num(column_names{c}(3:end-2))));
inds_int_in_file = cat(1, inds_int_in_file, c);
end
if(strfind(column_names{c}, '_c') > 0)
aus_pred_class = cat(1, aus_pred_class, int32(str2num(column_names{c}(3:end-2))));
inds_class_in_file = cat(1, inds_class_in_file, c);
end
end
%%
inds_au_int = zeros(size(aus_SEMAINE));
inds_au_class = zeros(size(aus_SEMAINE));
for ind=1:numel(aus_SEMAINE)
if(~isempty(find(aus_pred_int==aus_SEMAINE(ind), 1)))
inds_au_int(ind) = find(aus_pred_int==aus_SEMAINE(ind));
end
end
for ind=1:numel(aus_SEMAINE)
if(~isempty(find(aus_pred_class==aus_SEMAINE(ind), 1)))
inds_au_class(ind) = find(aus_pred_class==aus_SEMAINE(ind));
end
end
preds_all_class = [];
preds_all_int = [];
for i=1:numel(devel_recs)
fname = [out_loc, devel_recs{i}, '.au.txt'];
preds = dlmread(fname, ',', 1, 0);
% Read all of the intensity AUs
preds_int = preds(vid_ids(i,1):vid_ids(i,2) - 1, inds_int_in_file);
% Read all of the classification AUs
preds_class = preds(vid_ids(i,1):vid_ids(i,2) - 1, inds_class_in_file);
preds_all_class = cat(1, preds_all_class, preds_class);
preds_all_int = cat(1, preds_all_int, preds_int);
end
%%
f = fopen('SEMAINE_valid_res.txt', 'w');
for au = 1:numel(aus_SEMAINE)
if(inds_au_int(au) ~= 0)
tp = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) >= 1);
fp = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) >= 1);
fn = sum(labels_gt(:,au) == 1 & preds_all_int(:, inds_au_int(au)) < 1);
tn = sum(labels_gt(:,au) == 0 & preds_all_int(:, inds_au_int(au)) < 1);
precision = tp./(tp+fp);
recall = tp./(tp+fn);
f1 = 2 * precision .* recall ./ (precision + recall);
fprintf(f, 'AU%d intensity, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_SEMAINE(au), precision, recall, f1);
end
if(inds_au_class(au) ~= 0)
tp = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 1);
fp = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 1);
fn = sum(labels_gt(:,au) == 1 & preds_all_class(:, inds_au_class(au)) == 0);
tn = sum(labels_gt(:,au) == 0 & preds_all_class(:, inds_au_class(au)) == 0);
precision = tp./(tp+fp);
recall = tp./(tp+fn);
f1 = 2 * precision .* recall ./ (precision + recall);
fprintf(f, 'AU%d class, Precision - %.3f, Recall - %.3f, F1 - %.3f\n', aus_SEMAINE(au), precision, recall, f1);
end
end
fclose(f);

51
matlab_runners/Action Unit Experiments/run_BP4D_devel_txt_intensity.m Normal file
View file

@ -0,0 +1,51 @@
clear
addpath('../data extraction/');
find_BP4D;
bp4d_loc = [BP4D_dir '../BP4D/BP4D-training/'];
out_loc = './out_bp4d_static_intensity/';
if(~exist(out_loc, 'dir'))
mkdir(out_loc);
end
features_exe = '"../C++ models/Release/AUPrediction.exe"';
% Go two levels deep
bp4d_dirs = devel_recs;
in_txt = './txt_files/';
parfor f1=1:numel(bp4d_dirs)
if(isdir([bp4d_loc, bp4d_dirs{f1}]))
bp4d_2_dirs = dir([bp4d_loc, bp4d_dirs{f1}]);
bp4d_2_dirs = bp4d_2_dirs(3:end);
f1_dir = bp4d_dirs{f1};
for f2=1:numel(bp4d_2_dirs)
command = [features_exe, ' -scaling 0.4 -auloc "./AU_predictors/AU_SVR_BP4D_static.txt" -fx 2000 -fy 2000 -rigid -asvid -simscale 0.7 -simsize 112 '];
f2_dir = bp4d_2_dirs(f2).name;
if(isdir([bp4d_loc, bp4d_dirs{f1}]))
curr_txt_file = [in_txt, f1_dir, '_', f2_dir, '.txt'];
name = [f1_dir '_' f2_dir];
output_file = [out_loc name '/'];
output_aus = [out_loc name '.au.txt'];
command = cat(2, command, [' -ftxt "' curr_txt_file '" -oausreg "' output_aus '"']);
end
dos(command);
end
end
end