r
/
sustaining_gazes
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Bosphorus experiments with the new interface, improved the results slightly.

This commit is contained in:
Tadas Baltrusaitis 2017-12-06 15:43:55 +00:00
parent 1d3f32b7df
commit 077a8489bc
7 changed files with 120 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/local/LandmarkDetector/include/LandmarkDetectorModel.h
View File

@ -93,15 +93,12 @@ public:
dlib::frontal_face_detector face_detector_HOG;
// Validate if the detected landmarks are correct using an SVR regressor
// Validate if the detected landmarks are correct using a predictor on detected landmarks
DetectionValidator landmark_validator;
// Indicating if landmark detection succeeded (based on SVR validator)
// Indicating if landmark detection succeeded (based on detection validator)
bool detection_success;
// Indicating if the tracking has been initialised (for video based tracking)
bool tracking_initialised;
// Representing how confident we are that tracking succeeds (0 - complete failure, 1 - perfect success)
double detection_certainty;
@ -174,8 +171,17 @@ public:
// Helper reading function
void Read_CLNF(string clnf_location);
// Allows to set initialization accross hierarchical models as well
bool IsInitialized() const { return tracking_initialised; }
void SetInitialized(bool initialized);
void SetDetectionSuccess(bool detection_success);
private:
// Indicating if the tracking has been initialised (for video based tracking)
bool tracking_initialised;
// the speedup of RLMS using precalculated KDE responses (described in Saragih 2011 RLMS paper)
map<int, cv::Mat_<float> > kde_resp_precalc;

19
lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp
View File

@ -215,10 +215,10 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
// and using a smaller search area
// Indicating that this is a first detection in video sequence or after restart
bool initial_detection = !clnf_model.tracking_initialised;
bool initial_detection = !clnf_model.IsInitialized();
// Only do it if there was a face detection at all
if(clnf_model.tracking_initialised)
if(clnf_model.IsInitialized())
{
// The area of interest search size will depend if the previous track was successful
@ -254,8 +254,8 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
// This is used for both detection (if it the tracking has not been initialised yet) or if the tracking failed (however we do this every n frames, for speed)
// This also has the effect of an attempt to reinitialise just after the tracking has failed, which is useful during large motions
if((!clnf_model.tracking_initialised && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0)
|| (clnf_model.tracking_initialised && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0))
if((!clnf_model.IsInitialized() && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0)
|| (clnf_model.IsInitialized() && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0))
{
cv::Rect_<double> bounding_box;
@ -290,7 +290,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
if(face_detection_success)
{
// Indicate that tracking has started as a face was detected
clnf_model.tracking_initialised = true;
clnf_model.SetInitialized(true);
// Keep track of old model values so that they can be restored if redetection fails
cv::Vec6d params_global_init = clnf_model.params_global;
@ -335,7 +335,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
}
// if the model has not been initialised yet class it as a failure
if(!clnf_model.tracking_initialised)
if(!clnf_model.IsInitialized())
{
clnf_model.failures_in_a_row++;
}
@ -343,7 +343,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
// un-initialise the tracking
if( clnf_model.failures_in_a_row > 100)
{
clnf_model.tracking_initialised = false;
clnf_model.SetInitialized(false);
}
return clnf_model.detection_success;
@ -359,7 +359,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
clnf_model.pdm.CalcParams(clnf_model.params_global, bounding_box, clnf_model.params_local);
// indicate that face was detected so initialisation is not necessary
clnf_model.tracking_initialised = true;
clnf_model.SetInitialized(true);
}
return DetectLandmarksInVideo(grayscale_image, clnf_model, params);
@ -463,6 +463,9 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_<uchar> &grayscale_i
clnf_model.hierarchical_models[part].landmark_likelihoods = best_landmark_likelihoods_h[part].clone();
}
// To indicate that tracking/detection started and the values are valid, we assume that there is a face in the bounding box
clnf_model.SetInitialized(true);
return best_success;
}

32
lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp
View File

@ -508,8 +508,8 @@ void CLNF::Read(string main_location)
detected_landmarks.create(2 * pdm.NumberOfPoints(), 1);
detected_landmarks.setTo(0);
detection_success = false;
tracking_initialised = false;
SetDetectionSuccess(false);
SetInitialized(false);
model_likelihood = -10; // very low
detection_certainty = 0; // very uncertain
@ -526,13 +526,35 @@ void CLNF::Read(string main_location)
}
void CLNF::SetDetectionSuccess(bool success)
{
this->detection_success = success;
for (size_t i = 0; i < hierarchical_models.size(); ++i)
{
hierarchical_models[i].SetDetectionSuccess(success);
}
}
void CLNF::SetInitialized(bool initialized)
{
this->tracking_initialised = initialized;
for (size_t i = 0; i < hierarchical_models.size(); ++i)
{
hierarchical_models[i].SetInitialized(initialized);
}
}
// Resetting the model (for a new video, or complet reinitialisation
void CLNF::Reset()
{
detected_landmarks.setTo(0);
detection_success = false;
tracking_initialised = false;
SetDetectionSuccess(false);
SetInitialized(false);
model_likelihood = -10; // very low
detection_certainty = 0; // very uncertain
@ -1084,7 +1106,7 @@ cv::Mat_<double> CLNF::GetShape(double fx, double fy, double cx, double cy) cons
cv::Mat_<double> outShape(n, 3, 0.0);
// If the tracking started (otherwise no point reporting 3D shape)
if(this->tracking_initialised)
if(this->IsInitialized())
{
cv::Mat_<double> shape3d(n * 3, 1);

34
matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt
View File

@ -1,17 +1,17 @@
AU1 class, Precision - 0.393, Recall - 0.727, F1 - 0.510
AU2 class, Precision - 0.266, Recall - 0.850, F1 - 0.405
AU4 class, Precision - 0.511, Recall - 0.874, F1 - 0.645
AU5 class, Precision - 0.294, Recall - 0.968, F1 - 0.451
AU6 class, Precision - 0.346, Recall - 0.833, F1 - 0.489
AU7 class, Precision - 0.793, Recall - 0.750, F1 - 0.771
AU9 class, Precision - 0.316, Recall - 0.960, F1 - 0.475
AU10 class, Precision - 0.349, Recall - 0.773, F1 - 0.481
AU12 class, Precision - 0.674, Recall - 0.864, F1 - 0.757
AU14 class, Precision - 0.183, Recall - 0.863, F1 - 0.302
AU15 class, Precision - 0.183, Recall - 0.851, F1 - 0.302
AU17 class, Precision - 0.293, Recall - 0.889, F1 - 0.441
AU20 class, Precision - 0.114, Recall - 0.930, F1 - 0.203
AU23 class, Precision - 0.107, Recall - 0.889, F1 - 0.191
AU25 class, Precision - 0.860, Recall - 0.873, F1 - 0.866
AU26 class, Precision - 0.359, Recall - 0.811, F1 - 0.498
AU45 class, Precision - 0.318, Recall - 0.771, F1 - 0.450
AU1 class, Precision - 0.434, Recall - 0.673, F1 - 0.528
AU2 class, Precision - 0.298, Recall - 0.818, F1 - 0.437
AU4 class, Precision - 0.564, Recall - 0.861, F1 - 0.681
AU5 class, Precision - 0.387, Recall - 0.915, F1 - 0.544
AU6 class, Precision - 0.355, Recall - 0.811, F1 - 0.494
AU7 class, Precision - 0.778, Recall - 0.783, F1 - 0.780
AU9 class, Precision - 0.370, Recall - 0.953, F1 - 0.533
AU10 class, Precision - 0.340, Recall - 0.788, F1 - 0.475
AU12 class, Precision - 0.690, Recall - 0.842, F1 - 0.758
AU14 class, Precision - 0.185, Recall - 0.881, F1 - 0.305
AU15 class, Precision - 0.171, Recall - 0.851, F1 - 0.285
AU17 class, Precision - 0.309, Recall - 0.861, F1 - 0.455
AU20 class, Precision - 0.130, Recall - 0.921, F1 - 0.228
AU23 class, Precision - 0.104, Recall - 0.837, F1 - 0.186
AU25 class, Precision - 0.869, Recall - 0.860, F1 - 0.865
AU26 class, Precision - 0.368, Recall - 0.809, F1 - 0.506
AU45 class, Precision - 0.367, Recall - 0.754, F1 - 0.494

34
matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt
View File

@ -1,17 +1,17 @@
AU1 intensity, Corr - 0.717, RMS - 0.892, CCC - 0.668
AU2 intensity, Corr - 0.696, RMS - 0.774, CCC - 0.625
AU4 intensity, Corr - 0.802, RMS - 0.603, CCC - 0.776
AU5 intensity, Corr - 0.747, RMS - 0.832, CCC - 0.640
AU6 intensity, Corr - 0.556, RMS - 0.735, CCC - 0.533
AU7 intensity, Corr - 0.831, RMS - 0.757, CCC - 0.804
AU9 intensity, Corr - 0.779, RMS - 0.551, CCC - 0.738
AU10 intensity, Corr - 0.495, RMS - 0.719, CCC - 0.475
AU12 intensity, Corr - 0.810, RMS - 0.714, CCC - 0.753
AU14 intensity, Corr - 0.348, RMS - 0.896, CCC - 0.280
AU15 intensity, Corr - 0.527, RMS - 0.538, CCC - 0.448
AU17 intensity, Corr - 0.561, RMS - 0.882, CCC - 0.484
AU20 intensity, Corr - 0.413, RMS - 0.880, CCC - 0.285
AU23 intensity, Corr - 0.354, RMS - 0.753, CCC - 0.268
AU25 intensity, Corr - 0.847, RMS - 0.818, CCC - 0.811
AU26 intensity, Corr - 0.514, RMS - 0.955, CCC - 0.465
AU45 intensity, Corr - 0.868, RMS - 0.550, CCC - 0.848
AU1 intensity, Corr - 0.712, RMS - 0.925, CCC - 0.652
AU2 intensity, Corr - 0.696, RMS - 0.772, CCC - 0.626
AU4 intensity, Corr - 0.797, RMS - 0.623, CCC - 0.764
AU5 intensity, Corr - 0.767, RMS - 0.740, CCC - 0.694
AU6 intensity, Corr - 0.541, RMS - 0.786, CCC - 0.506
AU7 intensity, Corr - 0.830, RMS - 0.750, CCC - 0.811
AU9 intensity, Corr - 0.763, RMS - 0.611, CCC - 0.701
AU10 intensity, Corr - 0.491, RMS - 0.759, CCC - 0.461
AU12 intensity, Corr - 0.804, RMS - 0.715, CCC - 0.766
AU14 intensity, Corr - 0.357, RMS - 0.931, CCC - 0.277
AU15 intensity, Corr - 0.516, RMS - 0.565, CCC - 0.431
AU17 intensity, Corr - 0.554, RMS - 0.893, CCC - 0.477
AU20 intensity, Corr - 0.411, RMS - 0.900, CCC - 0.277
AU23 intensity, Corr - 0.351, RMS - 0.736, CCC - 0.274
AU25 intensity, Corr - 0.846, RMS - 0.809, CCC - 0.822
AU26 intensity, Corr - 0.516, RMS - 0.995, CCC - 0.453
AU45 intensity, Corr - 0.840, RMS - 0.662, CCC - 0.791

139
matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m
View File

@ -7,10 +7,6 @@ addpath('./helpers');
find_Bosphorus;
out_loc = './out_bosph/';
if(~exist(out_loc, 'dir'))
mkdir(out_loc);
end
%%
executable = '"../../x64/Release/FaceLandmarkImg.exe"';
@ -22,8 +18,8 @@ parfor f1=1:numel(bosph_dirs)
command = executable;
input_dir = [Bosphorus_dir, '/BosphorusDB/BosphorusDB/', bosph_dirs(f1).name];
command = cat(2, command, [' -fdir "' input_dir '" -ofdir "' out_loc '"']);
command = cat(2, command, ' -multi_view 1 -wild -q');
command = cat(2, command, [' -fdir "' input_dir '" -out_dir "' out_loc '"']);
command = cat(2, command, ' -multi_view 1 -wild -aus ');
dos(command);
@ -37,64 +33,32 @@ aus_Bosph = [1, 2, 4, 5, 6, 7, 9, 10, 12, 14, 15, 17, 20, 23, 25, 26, 45];
%% Read the predicted values
% First read the first file to get the ids and line numbers
% au occurences
fid = fopen([out_loc, filenames{1}, '_det_0.pts']);
data = fgetl(fid);
ind = 0;
beg_ind = -1;
end_ind = -1;
aus_det = [];
aus_det_id = [];
%%
while ischar(data)
if(~isempty(findstr(data, 'au occurences:')))
num_occurences = str2num(data(numel('au occurences:')+1:end));
% Skip ahead two lines
data = fgetl(fid);
data = fgetl(fid);
ind = ind + 2;
beg_ind = ind;
end
if(beg_ind ~= -1 && end_ind == -1)
if(~isempty(findstr(data, '}')))
end_ind = ind;
else
d = strsplit(data, ' ');
aus_det = cat(1, aus_det, str2num(d{1}(3:end)));
aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1);
end
end
data = fgetl(fid);
ind = ind + 1;
% First read the first file to get the column ids
tab = readtable([out_loc, filenames{1}, '.csv']);
column_names = tab.Properties.VariableNames;
aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_c'));
aus_det_cell = column_names(aus_det_id);
aus_det = zeros(size(aus_det_cell));
for i=1:numel(aus_det)
aus_det(i) = str2num(aus_det_cell{i}(3:4));
end
fclose(fid);
%%
labels_pred = zeros(size(labels_gt));
for i=1:numel(filenames)
% Will need to read the relevant AUs only
if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file'))
fid = fopen([out_loc, filenames{i}, '_det_0.pts']);
for k=1:beg_ind
data = fgetl(fid);
all_params = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0);
% if multiple faces detected just take the first row
aus_pred = all_params(1, aus_det_id);
for k=1:numel(aus_det)
if(sum(aus_Bosph == aus_det(k))>0)
labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k);
end
for k=1:num_occurences
data = fgetl(fid);
if(sum(aus_Bosph == aus_det(k))>0)
d = strsplit(data, ' ');
labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2});
end
end
fclose(fid);
end
end
%%
@ -122,63 +86,32 @@ fclose(f);
%% Read the predicted values for intensities
% First read the first file to get the ids and line numbers
% au occurences
fid = fopen([out_loc, filenames{1}, '_det_0.pts']);
data = fgetl(fid);
ind = 0;
beg_ind = -1;
end_ind = -1;
aus_det = [];
aus_det_id = [];
while ischar(data)
if(~isempty(findstr(data, 'au intensities:')))
num_occurences = str2num(data(numel('au intensities:')+1:end));
% Skip ahead two lines
data = fgetl(fid);
data = fgetl(fid);
ind = ind + 2;
beg_ind = ind;
end
if(beg_ind ~= -1 && end_ind == -1)
if(~isempty(findstr(data, '}')))
end_ind = ind;
else
d = strsplit(data, ' ');
aus_det = cat(1, aus_det, str2num(d{1}(3:end)));
aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1);
end
end
data = fgetl(fid);
ind = ind + 1;
% First read the first file to get the column ids
tab = readtable([out_loc, filenames{1}, '.csv']);
column_names = tab.Properties.VariableNames;
aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r'));
aus_det_cell = column_names(aus_det_id);
aus_det = zeros(size(aus_det_cell));
for i=1:numel(aus_det)
aus_det(i) = str2num(aus_det_cell{i}(3:4));
end
fclose(fid);
%%
labels_pred = zeros(size(labels_gt));
for i=1:numel(filenames)
% Will need to read the relevant AUs only
if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file'))
fid = fopen([out_loc, filenames{i}, '_det_0.pts']);
for k=1:beg_ind
data = fgetl(fid);
all_params = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0);
% if multiple faces detected just take the first row
aus_pred = all_params(1, aus_det_id);
for k=1:numel(aus_det)
if(sum(aus_Bosph == aus_det(k))>0)
labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k);
end
for k=1:num_occurences
data = fgetl(fid);
if(sum(aus_Bosph == aus_det(k))>0)
d = strsplit(data, ' ');
labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2});
end
end
fclose(fid);
end
end
%%

2
matlab_runners/Full_test_suite.m
View File

@ -23,7 +23,7 @@ cd('../');
cd('Action Unit Experiments');
run_AU_prediction_Bosphorus
assert(mean(cccs_reg) > 0.56);
assert(mean(f1s_class) > 0.46);
assert(mean(f1s_class) > 0.49);
run_AU_prediction_BP4D
assert(mean(ints_cccs) > 0.6);