Bosphorus experiments with the new interface, improved the results slightly.
This commit is contained in:
Tadas Baltrusaitis
parent
1d3f32b7df
commit
077a8489bc
7 changed files with 120 additions and 156 deletions
|
|
@ -93,15 +93,12 @@ public:
|
|||
dlib::frontal_face_detector face_detector_HOG;
|
||||
|
||||
|
||||
// Validate if the detected landmarks are correct using an SVR regressor
|
||||
// Validate if the detected landmarks are correct using a predictor on detected landmarks
|
||||
DetectionValidator landmark_validator;
|
||||
|
||||
// Indicating if landmark detection succeeded (based on SVR validator)
|
||||
// Indicating if landmark detection succeeded (based on detection validator)
|
||||
bool detection_success;
|
||||
|
||||
// Indicating if the tracking has been initialised (for video based tracking)
|
||||
bool tracking_initialised;
|
||||
|
||||
// Representing how confident we are that tracking succeeds (0 - complete failure, 1 - perfect success)
|
||||
double detection_certainty;
|
||||
|
||||
|
|
@ -174,8 +171,17 @@ public:
|
|||
// Helper reading function
|
||||
void Read_CLNF(string clnf_location);
|
||||
|
||||
// Allows to set initialization accross hierarchical models as well
|
||||
bool IsInitialized() const { return tracking_initialised; }
|
||||
void SetInitialized(bool initialized);
|
||||
void SetDetectionSuccess(bool detection_success);
|
||||
|
||||
private:
|
||||
|
||||
|
||||
// Indicating if the tracking has been initialised (for video based tracking)
|
||||
bool tracking_initialised;
|
||||
|
||||
// the speedup of RLMS using precalculated KDE responses (described in Saragih 2011 RLMS paper)
|
||||
map<int, cv::Mat_<float> > kde_resp_precalc;
|
||||
|
||||
|
|
|
|||
|
|
@ -215,10 +215,10 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
|
|||
// and using a smaller search area
|
||||
|
||||
// Indicating that this is a first detection in video sequence or after restart
|
||||
bool initial_detection = !clnf_model.tracking_initialised;
|
||||
bool initial_detection = !clnf_model.IsInitialized();
|
||||
|
||||
// Only do it if there was a face detection at all
|
||||
if(clnf_model.tracking_initialised)
|
||||
if(clnf_model.IsInitialized())
|
||||
{
|
||||
|
||||
// The area of interest search size will depend if the previous track was successful
|
||||
|
|
@ -254,8 +254,8 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
|
|||
|
||||
// This is used for both detection (if it the tracking has not been initialised yet) or if the tracking failed (however we do this every n frames, for speed)
|
||||
// This also has the effect of an attempt to reinitialise just after the tracking has failed, which is useful during large motions
|
||||
if((!clnf_model.tracking_initialised && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0)
|
||||
|| (clnf_model.tracking_initialised && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0))
|
||||
if((!clnf_model.IsInitialized() && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0)
|
||||
|| (clnf_model.IsInitialized() && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0))
|
||||
{
|
||||
|
||||
cv::Rect_<double> bounding_box;
|
||||
|
|
@ -290,7 +290,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
|
|||
if(face_detection_success)
|
||||
{
|
||||
// Indicate that tracking has started as a face was detected
|
||||
clnf_model.tracking_initialised = true;
|
||||
clnf_model.SetInitialized(true);
|
||||
|
||||
// Keep track of old model values so that they can be restored if redetection fails
|
||||
cv::Vec6d params_global_init = clnf_model.params_global;
|
||||
|
|
@ -335,7 +335,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
|
|||
}
|
||||
|
||||
// if the model has not been initialised yet class it as a failure
|
||||
if(!clnf_model.tracking_initialised)
|
||||
if(!clnf_model.IsInitialized())
|
||||
{
|
||||
clnf_model.failures_in_a_row++;
|
||||
}
|
||||
|
|
@ -343,7 +343,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
|
|||
// un-initialise the tracking
|
||||
if( clnf_model.failures_in_a_row > 100)
|
||||
{
|
||||
clnf_model.tracking_initialised = false;
|
||||
clnf_model.SetInitialized(false);
|
||||
}
|
||||
|
||||
return clnf_model.detection_success;
|
||||
|
|
@ -359,7 +359,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
|
|||
clnf_model.pdm.CalcParams(clnf_model.params_global, bounding_box, clnf_model.params_local);
|
||||
|
||||
// indicate that face was detected so initialisation is not necessary
|
||||
clnf_model.tracking_initialised = true;
|
||||
clnf_model.SetInitialized(true);
|
||||
}
|
||||
|
||||
return DetectLandmarksInVideo(grayscale_image, clnf_model, params);
|
||||
|
|
@ -463,6 +463,9 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_<uchar> &grayscale_i
|
|||
clnf_model.hierarchical_models[part].landmark_likelihoods = best_landmark_likelihoods_h[part].clone();
|
||||
}
|
||||
|
||||
// To indicate that tracking/detection started and the values are valid, we assume that there is a face in the bounding box
|
||||
clnf_model.SetInitialized(true);
|
||||
|
||||
return best_success;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -508,8 +508,8 @@ void CLNF::Read(string main_location)
|
|||
detected_landmarks.create(2 * pdm.NumberOfPoints(), 1);
|
||||
detected_landmarks.setTo(0);
|
||||
|
||||
detection_success = false;
|
||||
tracking_initialised = false;
|
||||
SetDetectionSuccess(false);
|
||||
SetInitialized(false);
|
||||
model_likelihood = -10; // very low
|
||||
detection_certainty = 0; // very uncertain
|
||||
|
||||
|
|
@ -526,13 +526,35 @@ void CLNF::Read(string main_location)
|
|||
|
||||
}
|
||||
|
||||
void CLNF::SetDetectionSuccess(bool success)
|
||||
{
|
||||
this->detection_success = success;
|
||||
|
||||
for (size_t i = 0; i < hierarchical_models.size(); ++i)
|
||||
{
|
||||
hierarchical_models[i].SetDetectionSuccess(success);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void CLNF::SetInitialized(bool initialized)
|
||||
{
|
||||
this->tracking_initialised = initialized;
|
||||
|
||||
for (size_t i = 0; i < hierarchical_models.size(); ++i)
|
||||
{
|
||||
hierarchical_models[i].SetInitialized(initialized);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Resetting the model (for a new video, or complet reinitialisation
|
||||
void CLNF::Reset()
|
||||
{
|
||||
detected_landmarks.setTo(0);
|
||||
|
||||
detection_success = false;
|
||||
tracking_initialised = false;
|
||||
SetDetectionSuccess(false);
|
||||
SetInitialized(false);
|
||||
model_likelihood = -10; // very low
|
||||
detection_certainty = 0; // very uncertain
|
||||
|
||||
|
|
@ -1084,7 +1106,7 @@ cv::Mat_<double> CLNF::GetShape(double fx, double fy, double cx, double cy) cons
|
|||
cv::Mat_<double> outShape(n, 3, 0.0);
|
||||
|
||||
// If the tracking started (otherwise no point reporting 3D shape)
|
||||
if(this->tracking_initialised)
|
||||
if(this->IsInitialized())
|
||||
{
|
||||
|
||||
cv::Mat_<double> shape3d(n * 3, 1);
|
||||
|
|
|
|||
|
|
@ -1,17 +1,17 @@
|
|||
AU1 class, Precision - 0.393, Recall - 0.727, F1 - 0.510
|
||||
AU2 class, Precision - 0.266, Recall - 0.850, F1 - 0.405
|
||||
AU4 class, Precision - 0.511, Recall - 0.874, F1 - 0.645
|
||||
AU5 class, Precision - 0.294, Recall - 0.968, F1 - 0.451
|
||||
AU6 class, Precision - 0.346, Recall - 0.833, F1 - 0.489
|
||||
AU7 class, Precision - 0.793, Recall - 0.750, F1 - 0.771
|
||||
AU9 class, Precision - 0.316, Recall - 0.960, F1 - 0.475
|
||||
AU10 class, Precision - 0.349, Recall - 0.773, F1 - 0.481
|
||||
AU12 class, Precision - 0.674, Recall - 0.864, F1 - 0.757
|
||||
AU14 class, Precision - 0.183, Recall - 0.863, F1 - 0.302
|
||||
AU15 class, Precision - 0.183, Recall - 0.851, F1 - 0.302
|
||||
AU17 class, Precision - 0.293, Recall - 0.889, F1 - 0.441
|
||||
AU20 class, Precision - 0.114, Recall - 0.930, F1 - 0.203
|
||||
AU23 class, Precision - 0.107, Recall - 0.889, F1 - 0.191
|
||||
AU25 class, Precision - 0.860, Recall - 0.873, F1 - 0.866
|
||||
AU26 class, Precision - 0.359, Recall - 0.811, F1 - 0.498
|
||||
AU45 class, Precision - 0.318, Recall - 0.771, F1 - 0.450
|
||||
AU1 class, Precision - 0.434, Recall - 0.673, F1 - 0.528
|
||||
AU2 class, Precision - 0.298, Recall - 0.818, F1 - 0.437
|
||||
AU4 class, Precision - 0.564, Recall - 0.861, F1 - 0.681
|
||||
AU5 class, Precision - 0.387, Recall - 0.915, F1 - 0.544
|
||||
AU6 class, Precision - 0.355, Recall - 0.811, F1 - 0.494
|
||||
AU7 class, Precision - 0.778, Recall - 0.783, F1 - 0.780
|
||||
AU9 class, Precision - 0.370, Recall - 0.953, F1 - 0.533
|
||||
AU10 class, Precision - 0.340, Recall - 0.788, F1 - 0.475
|
||||
AU12 class, Precision - 0.690, Recall - 0.842, F1 - 0.758
|
||||
AU14 class, Precision - 0.185, Recall - 0.881, F1 - 0.305
|
||||
AU15 class, Precision - 0.171, Recall - 0.851, F1 - 0.285
|
||||
AU17 class, Precision - 0.309, Recall - 0.861, F1 - 0.455
|
||||
AU20 class, Precision - 0.130, Recall - 0.921, F1 - 0.228
|
||||
AU23 class, Precision - 0.104, Recall - 0.837, F1 - 0.186
|
||||
AU25 class, Precision - 0.869, Recall - 0.860, F1 - 0.865
|
||||
AU26 class, Precision - 0.368, Recall - 0.809, F1 - 0.506
|
||||
AU45 class, Precision - 0.367, Recall - 0.754, F1 - 0.494
|
||||
|
|
|
|||
|
|
@ -1,17 +1,17 @@
|
|||
AU1 intensity, Corr - 0.717, RMS - 0.892, CCC - 0.668
|
||||
AU2 intensity, Corr - 0.696, RMS - 0.774, CCC - 0.625
|
||||
AU4 intensity, Corr - 0.802, RMS - 0.603, CCC - 0.776
|
||||
AU5 intensity, Corr - 0.747, RMS - 0.832, CCC - 0.640
|
||||
AU6 intensity, Corr - 0.556, RMS - 0.735, CCC - 0.533
|
||||
AU7 intensity, Corr - 0.831, RMS - 0.757, CCC - 0.804
|
||||
AU9 intensity, Corr - 0.779, RMS - 0.551, CCC - 0.738
|
||||
AU10 intensity, Corr - 0.495, RMS - 0.719, CCC - 0.475
|
||||
AU12 intensity, Corr - 0.810, RMS - 0.714, CCC - 0.753
|
||||
AU14 intensity, Corr - 0.348, RMS - 0.896, CCC - 0.280
|
||||
AU15 intensity, Corr - 0.527, RMS - 0.538, CCC - 0.448
|
||||
AU17 intensity, Corr - 0.561, RMS - 0.882, CCC - 0.484
|
||||
AU20 intensity, Corr - 0.413, RMS - 0.880, CCC - 0.285
|
||||
AU23 intensity, Corr - 0.354, RMS - 0.753, CCC - 0.268
|
||||
AU25 intensity, Corr - 0.847, RMS - 0.818, CCC - 0.811
|
||||
AU26 intensity, Corr - 0.514, RMS - 0.955, CCC - 0.465
|
||||
AU45 intensity, Corr - 0.868, RMS - 0.550, CCC - 0.848
|
||||
AU1 intensity, Corr - 0.712, RMS - 0.925, CCC - 0.652
|
||||
AU2 intensity, Corr - 0.696, RMS - 0.772, CCC - 0.626
|
||||
AU4 intensity, Corr - 0.797, RMS - 0.623, CCC - 0.764
|
||||
AU5 intensity, Corr - 0.767, RMS - 0.740, CCC - 0.694
|
||||
AU6 intensity, Corr - 0.541, RMS - 0.786, CCC - 0.506
|
||||
AU7 intensity, Corr - 0.830, RMS - 0.750, CCC - 0.811
|
||||
AU9 intensity, Corr - 0.763, RMS - 0.611, CCC - 0.701
|
||||
AU10 intensity, Corr - 0.491, RMS - 0.759, CCC - 0.461
|
||||
AU12 intensity, Corr - 0.804, RMS - 0.715, CCC - 0.766
|
||||
AU14 intensity, Corr - 0.357, RMS - 0.931, CCC - 0.277
|
||||
AU15 intensity, Corr - 0.516, RMS - 0.565, CCC - 0.431
|
||||
AU17 intensity, Corr - 0.554, RMS - 0.893, CCC - 0.477
|
||||
AU20 intensity, Corr - 0.411, RMS - 0.900, CCC - 0.277
|
||||
AU23 intensity, Corr - 0.351, RMS - 0.736, CCC - 0.274
|
||||
AU25 intensity, Corr - 0.846, RMS - 0.809, CCC - 0.822
|
||||
AU26 intensity, Corr - 0.516, RMS - 0.995, CCC - 0.453
|
||||
AU45 intensity, Corr - 0.840, RMS - 0.662, CCC - 0.791
|
||||
|
|
|
|||
|
|
@ -7,10 +7,6 @@ addpath('./helpers');
|
|||
find_Bosphorus;
|
||||
out_loc = './out_bosph/';
|
||||
|
||||
if(~exist(out_loc, 'dir'))
|
||||
mkdir(out_loc);
|
||||
end
|
||||
|
||||
%%
|
||||
executable = '"../../x64/Release/FaceLandmarkImg.exe"';
|
||||
|
||||
|
|
@ -22,8 +18,8 @@ parfor f1=1:numel(bosph_dirs)
|
|||
command = executable;
|
||||
|
||||
input_dir = [Bosphorus_dir, '/BosphorusDB/BosphorusDB/', bosph_dirs(f1).name];
|
||||
command = cat(2, command, [' -fdir "' input_dir '" -ofdir "' out_loc '"']);
|
||||
command = cat(2, command, ' -multi_view 1 -wild -q');
|
||||
command = cat(2, command, [' -fdir "' input_dir '" -out_dir "' out_loc '"']);
|
||||
command = cat(2, command, ' -multi_view 1 -wild -aus ');
|
||||
|
||||
dos(command);
|
||||
|
||||
|
|
@ -37,64 +33,32 @@ aus_Bosph = [1, 2, 4, 5, 6, 7, 9, 10, 12, 14, 15, 17, 20, 23, 25, 26, 45];
|
|||
|
||||
%% Read the predicted values
|
||||
|
||||
% First read the first file to get the ids and line numbers
|
||||
% au occurences
|
||||
fid = fopen([out_loc, filenames{1}, '_det_0.pts']);
|
||||
data = fgetl(fid);
|
||||
|
||||
ind = 0;
|
||||
beg_ind = -1;
|
||||
end_ind = -1;
|
||||
aus_det = [];
|
||||
aus_det_id = [];
|
||||
|
||||
%%
|
||||
while ischar(data)
|
||||
if(~isempty(findstr(data, 'au occurences:')))
|
||||
num_occurences = str2num(data(numel('au occurences:')+1:end));
|
||||
% Skip ahead two lines
|
||||
data = fgetl(fid);
|
||||
data = fgetl(fid);
|
||||
ind = ind + 2;
|
||||
beg_ind = ind;
|
||||
end
|
||||
|
||||
if(beg_ind ~= -1 && end_ind == -1)
|
||||
if(~isempty(findstr(data, '}')))
|
||||
end_ind = ind;
|
||||
else
|
||||
d = strsplit(data, ' ');
|
||||
aus_det = cat(1, aus_det, str2num(d{1}(3:end)));
|
||||
aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1);
|
||||
end
|
||||
end
|
||||
|
||||
data = fgetl(fid);
|
||||
ind = ind + 1;
|
||||
% First read the first file to get the column ids
|
||||
tab = readtable([out_loc, filenames{1}, '.csv']);
|
||||
column_names = tab.Properties.VariableNames;
|
||||
aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_c'));
|
||||
aus_det_cell = column_names(aus_det_id);
|
||||
aus_det = zeros(size(aus_det_cell));
|
||||
for i=1:numel(aus_det)
|
||||
aus_det(i) = str2num(aus_det_cell{i}(3:4));
|
||||
end
|
||||
fclose(fid);
|
||||
|
||||
%%
|
||||
labels_pred = zeros(size(labels_gt));
|
||||
for i=1:numel(filenames)
|
||||
|
||||
% Will need to read the relevant AUs only
|
||||
if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file'))
|
||||
fid = fopen([out_loc, filenames{i}, '_det_0.pts']);
|
||||
for k=1:beg_ind
|
||||
data = fgetl(fid);
|
||||
end
|
||||
all_params = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0);
|
||||
|
||||
for k=1:num_occurences
|
||||
data = fgetl(fid);
|
||||
% if multiple faces detected just take the first row
|
||||
aus_pred = all_params(1, aus_det_id);
|
||||
|
||||
for k=1:numel(aus_det)
|
||||
if(sum(aus_Bosph == aus_det(k))>0)
|
||||
d = strsplit(data, ' ');
|
||||
labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2});
|
||||
labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k);
|
||||
end
|
||||
end
|
||||
|
||||
fclose(fid);
|
||||
end
|
||||
end
|
||||
|
||||
%%
|
||||
|
|
@ -122,63 +86,32 @@ fclose(f);
|
|||
|
||||
%% Read the predicted values for intensities
|
||||
|
||||
% First read the first file to get the ids and line numbers
|
||||
% au occurences
|
||||
fid = fopen([out_loc, filenames{1}, '_det_0.pts']);
|
||||
data = fgetl(fid);
|
||||
|
||||
ind = 0;
|
||||
beg_ind = -1;
|
||||
end_ind = -1;
|
||||
aus_det = [];
|
||||
aus_det_id = [];
|
||||
|
||||
while ischar(data)
|
||||
if(~isempty(findstr(data, 'au intensities:')))
|
||||
num_occurences = str2num(data(numel('au intensities:')+1:end));
|
||||
% Skip ahead two lines
|
||||
data = fgetl(fid);
|
||||
data = fgetl(fid);
|
||||
ind = ind + 2;
|
||||
beg_ind = ind;
|
||||
end
|
||||
|
||||
if(beg_ind ~= -1 && end_ind == -1)
|
||||
if(~isempty(findstr(data, '}')))
|
||||
end_ind = ind;
|
||||
else
|
||||
d = strsplit(data, ' ');
|
||||
aus_det = cat(1, aus_det, str2num(d{1}(3:end)));
|
||||
aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1);
|
||||
end
|
||||
end
|
||||
|
||||
data = fgetl(fid);
|
||||
ind = ind + 1;
|
||||
% First read the first file to get the column ids
|
||||
tab = readtable([out_loc, filenames{1}, '.csv']);
|
||||
column_names = tab.Properties.VariableNames;
|
||||
aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r'));
|
||||
aus_det_cell = column_names(aus_det_id);
|
||||
aus_det = zeros(size(aus_det_cell));
|
||||
for i=1:numel(aus_det)
|
||||
aus_det(i) = str2num(aus_det_cell{i}(3:4));
|
||||
end
|
||||
fclose(fid);
|
||||
|
||||
%%
|
||||
labels_pred = zeros(size(labels_gt));
|
||||
for i=1:numel(filenames)
|
||||
|
||||
% Will need to read the relevant AUs only
|
||||
if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file'))
|
||||
fid = fopen([out_loc, filenames{i}, '_det_0.pts']);
|
||||
for k=1:beg_ind
|
||||
data = fgetl(fid);
|
||||
end
|
||||
all_params = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0);
|
||||
|
||||
for k=1:num_occurences
|
||||
data = fgetl(fid);
|
||||
% if multiple faces detected just take the first row
|
||||
aus_pred = all_params(1, aus_det_id);
|
||||
|
||||
for k=1:numel(aus_det)
|
||||
if(sum(aus_Bosph == aus_det(k))>0)
|
||||
d = strsplit(data, ' ');
|
||||
labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2});
|
||||
labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k);
|
||||
end
|
||||
end
|
||||
|
||||
fclose(fid);
|
||||
end
|
||||
end
|
||||
|
||||
%%
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ cd('../');
|
|||
cd('Action Unit Experiments');
|
||||
run_AU_prediction_Bosphorus
|
||||
assert(mean(cccs_reg) > 0.56);
|
||||
assert(mean(f1s_class) > 0.46);
|
||||
assert(mean(f1s_class) > 0.49);
|
||||
|
||||
run_AU_prediction_BP4D
|
||||
assert(mean(ints_cccs) > 0.6);
|
||||
|
|
|
|||
Loading…
Reference in a new issue