From 077a8489bcba33f5e7cf2467e6a198402c69305d Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Wed, 6 Dec 2017 15:43:55 +0000 Subject: [PATCH] Bosphorus experiments with the new interface, improved the results slightly. --- .../include/LandmarkDetectorModel.h | 16 +- .../src/LandmarkDetectorFunc.cpp | 19 ++- .../src/LandmarkDetectorModel.cpp | 32 +++- .../results/Bosphorus_res_class.txt | 34 ++--- .../results/Bosphorus_res_int.txt | 34 ++--- .../run_AU_prediction_Bosphorus.m | 139 +++++------------- matlab_runners/Full_test_suite.m | 2 +- 7 files changed, 120 insertions(+), 156 deletions(-) diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h index eae4337..65bb587 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h @@ -93,15 +93,12 @@ public: dlib::frontal_face_detector face_detector_HOG; - // Validate if the detected landmarks are correct using an SVR regressor + // Validate if the detected landmarks are correct using a predictor on detected landmarks DetectionValidator landmark_validator; - // Indicating if landmark detection succeeded (based on SVR validator) + // Indicating if landmark detection succeeded (based on detection validator) bool detection_success; - // Indicating if the tracking has been initialised (for video based tracking) - bool tracking_initialised; - // Representing how confident we are that tracking succeeds (0 - complete failure, 1 - perfect success) double detection_certainty; @@ -174,8 +171,17 @@ public: // Helper reading function void Read_CLNF(string clnf_location); + // Allows to set initialization accross hierarchical models as well + bool IsInitialized() const { return tracking_initialised; } + void SetInitialized(bool initialized); + void SetDetectionSuccess(bool detection_success); + private: + + // Indicating if the tracking has been initialised (for video based tracking) + bool tracking_initialised; + // the speedup of RLMS using precalculated KDE responses (described in Saragih 2011 RLMS paper) map > kde_resp_precalc; diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp index 7a267e1..1c9f802 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp @@ -215,10 +215,10 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i // and using a smaller search area // Indicating that this is a first detection in video sequence or after restart - bool initial_detection = !clnf_model.tracking_initialised; + bool initial_detection = !clnf_model.IsInitialized(); // Only do it if there was a face detection at all - if(clnf_model.tracking_initialised) + if(clnf_model.IsInitialized()) { // The area of interest search size will depend if the previous track was successful @@ -254,8 +254,8 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i // This is used for both detection (if it the tracking has not been initialised yet) or if the tracking failed (however we do this every n frames, for speed) // This also has the effect of an attempt to reinitialise just after the tracking has failed, which is useful during large motions - if((!clnf_model.tracking_initialised && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0) - || (clnf_model.tracking_initialised && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0)) + if((!clnf_model.IsInitialized() && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0) + || (clnf_model.IsInitialized() && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0)) { cv::Rect_ bounding_box; @@ -290,7 +290,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i if(face_detection_success) { // Indicate that tracking has started as a face was detected - clnf_model.tracking_initialised = true; + clnf_model.SetInitialized(true); // Keep track of old model values so that they can be restored if redetection fails cv::Vec6d params_global_init = clnf_model.params_global; @@ -335,7 +335,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i } // if the model has not been initialised yet class it as a failure - if(!clnf_model.tracking_initialised) + if(!clnf_model.IsInitialized()) { clnf_model.failures_in_a_row++; } @@ -343,7 +343,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i // un-initialise the tracking if( clnf_model.failures_in_a_row > 100) { - clnf_model.tracking_initialised = false; + clnf_model.SetInitialized(false); } return clnf_model.detection_success; @@ -359,7 +359,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i clnf_model.pdm.CalcParams(clnf_model.params_global, bounding_box, clnf_model.params_local); // indicate that face was detected so initialisation is not necessary - clnf_model.tracking_initialised = true; + clnf_model.SetInitialized(true); } return DetectLandmarksInVideo(grayscale_image, clnf_model, params); @@ -463,6 +463,9 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_ &grayscale_i clnf_model.hierarchical_models[part].landmark_likelihoods = best_landmark_likelihoods_h[part].clone(); } + // To indicate that tracking/detection started and the values are valid, we assume that there is a face in the bounding box + clnf_model.SetInitialized(true); + return best_success; } diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index 0613c11..441b3f3 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -508,8 +508,8 @@ void CLNF::Read(string main_location) detected_landmarks.create(2 * pdm.NumberOfPoints(), 1); detected_landmarks.setTo(0); - detection_success = false; - tracking_initialised = false; + SetDetectionSuccess(false); + SetInitialized(false); model_likelihood = -10; // very low detection_certainty = 0; // very uncertain @@ -526,13 +526,35 @@ void CLNF::Read(string main_location) } +void CLNF::SetDetectionSuccess(bool success) +{ + this->detection_success = success; + + for (size_t i = 0; i < hierarchical_models.size(); ++i) + { + hierarchical_models[i].SetDetectionSuccess(success); + } + +} + +void CLNF::SetInitialized(bool initialized) +{ + this->tracking_initialised = initialized; + + for (size_t i = 0; i < hierarchical_models.size(); ++i) + { + hierarchical_models[i].SetInitialized(initialized); + } + +} + // Resetting the model (for a new video, or complet reinitialisation void CLNF::Reset() { detected_landmarks.setTo(0); - detection_success = false; - tracking_initialised = false; + SetDetectionSuccess(false); + SetInitialized(false); model_likelihood = -10; // very low detection_certainty = 0; // very uncertain @@ -1084,7 +1106,7 @@ cv::Mat_ CLNF::GetShape(double fx, double fy, double cx, double cy) cons cv::Mat_ outShape(n, 3, 0.0); // If the tracking started (otherwise no point reporting 3D shape) - if(this->tracking_initialised) + if(this->IsInitialized()) { cv::Mat_ shape3d(n * 3, 1); diff --git a/matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt b/matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt index 2952f05..d7792c0 100644 --- a/matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt +++ b/matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt @@ -1,17 +1,17 @@ -AU1 class, Precision - 0.393, Recall - 0.727, F1 - 0.510 -AU2 class, Precision - 0.266, Recall - 0.850, F1 - 0.405 -AU4 class, Precision - 0.511, Recall - 0.874, F1 - 0.645 -AU5 class, Precision - 0.294, Recall - 0.968, F1 - 0.451 -AU6 class, Precision - 0.346, Recall - 0.833, F1 - 0.489 -AU7 class, Precision - 0.793, Recall - 0.750, F1 - 0.771 -AU9 class, Precision - 0.316, Recall - 0.960, F1 - 0.475 -AU10 class, Precision - 0.349, Recall - 0.773, F1 - 0.481 -AU12 class, Precision - 0.674, Recall - 0.864, F1 - 0.757 -AU14 class, Precision - 0.183, Recall - 0.863, F1 - 0.302 -AU15 class, Precision - 0.183, Recall - 0.851, F1 - 0.302 -AU17 class, Precision - 0.293, Recall - 0.889, F1 - 0.441 -AU20 class, Precision - 0.114, Recall - 0.930, F1 - 0.203 -AU23 class, Precision - 0.107, Recall - 0.889, F1 - 0.191 -AU25 class, Precision - 0.860, Recall - 0.873, F1 - 0.866 -AU26 class, Precision - 0.359, Recall - 0.811, F1 - 0.498 -AU45 class, Precision - 0.318, Recall - 0.771, F1 - 0.450 +AU1 class, Precision - 0.434, Recall - 0.673, F1 - 0.528 +AU2 class, Precision - 0.298, Recall - 0.818, F1 - 0.437 +AU4 class, Precision - 0.564, Recall - 0.861, F1 - 0.681 +AU5 class, Precision - 0.387, Recall - 0.915, F1 - 0.544 +AU6 class, Precision - 0.355, Recall - 0.811, F1 - 0.494 +AU7 class, Precision - 0.778, Recall - 0.783, F1 - 0.780 +AU9 class, Precision - 0.370, Recall - 0.953, F1 - 0.533 +AU10 class, Precision - 0.340, Recall - 0.788, F1 - 0.475 +AU12 class, Precision - 0.690, Recall - 0.842, F1 - 0.758 +AU14 class, Precision - 0.185, Recall - 0.881, F1 - 0.305 +AU15 class, Precision - 0.171, Recall - 0.851, F1 - 0.285 +AU17 class, Precision - 0.309, Recall - 0.861, F1 - 0.455 +AU20 class, Precision - 0.130, Recall - 0.921, F1 - 0.228 +AU23 class, Precision - 0.104, Recall - 0.837, F1 - 0.186 +AU25 class, Precision - 0.869, Recall - 0.860, F1 - 0.865 +AU26 class, Precision - 0.368, Recall - 0.809, F1 - 0.506 +AU45 class, Precision - 0.367, Recall - 0.754, F1 - 0.494 diff --git a/matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt b/matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt index 7ea7bf3..8ac0ee9 100644 --- a/matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt +++ b/matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt @@ -1,17 +1,17 @@ -AU1 intensity, Corr - 0.717, RMS - 0.892, CCC - 0.668 -AU2 intensity, Corr - 0.696, RMS - 0.774, CCC - 0.625 -AU4 intensity, Corr - 0.802, RMS - 0.603, CCC - 0.776 -AU5 intensity, Corr - 0.747, RMS - 0.832, CCC - 0.640 -AU6 intensity, Corr - 0.556, RMS - 0.735, CCC - 0.533 -AU7 intensity, Corr - 0.831, RMS - 0.757, CCC - 0.804 -AU9 intensity, Corr - 0.779, RMS - 0.551, CCC - 0.738 -AU10 intensity, Corr - 0.495, RMS - 0.719, CCC - 0.475 -AU12 intensity, Corr - 0.810, RMS - 0.714, CCC - 0.753 -AU14 intensity, Corr - 0.348, RMS - 0.896, CCC - 0.280 -AU15 intensity, Corr - 0.527, RMS - 0.538, CCC - 0.448 -AU17 intensity, Corr - 0.561, RMS - 0.882, CCC - 0.484 -AU20 intensity, Corr - 0.413, RMS - 0.880, CCC - 0.285 -AU23 intensity, Corr - 0.354, RMS - 0.753, CCC - 0.268 -AU25 intensity, Corr - 0.847, RMS - 0.818, CCC - 0.811 -AU26 intensity, Corr - 0.514, RMS - 0.955, CCC - 0.465 -AU45 intensity, Corr - 0.868, RMS - 0.550, CCC - 0.848 +AU1 intensity, Corr - 0.712, RMS - 0.925, CCC - 0.652 +AU2 intensity, Corr - 0.696, RMS - 0.772, CCC - 0.626 +AU4 intensity, Corr - 0.797, RMS - 0.623, CCC - 0.764 +AU5 intensity, Corr - 0.767, RMS - 0.740, CCC - 0.694 +AU6 intensity, Corr - 0.541, RMS - 0.786, CCC - 0.506 +AU7 intensity, Corr - 0.830, RMS - 0.750, CCC - 0.811 +AU9 intensity, Corr - 0.763, RMS - 0.611, CCC - 0.701 +AU10 intensity, Corr - 0.491, RMS - 0.759, CCC - 0.461 +AU12 intensity, Corr - 0.804, RMS - 0.715, CCC - 0.766 +AU14 intensity, Corr - 0.357, RMS - 0.931, CCC - 0.277 +AU15 intensity, Corr - 0.516, RMS - 0.565, CCC - 0.431 +AU17 intensity, Corr - 0.554, RMS - 0.893, CCC - 0.477 +AU20 intensity, Corr - 0.411, RMS - 0.900, CCC - 0.277 +AU23 intensity, Corr - 0.351, RMS - 0.736, CCC - 0.274 +AU25 intensity, Corr - 0.846, RMS - 0.809, CCC - 0.822 +AU26 intensity, Corr - 0.516, RMS - 0.995, CCC - 0.453 +AU45 intensity, Corr - 0.840, RMS - 0.662, CCC - 0.791 diff --git a/matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m b/matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m index 3c5e745..9762790 100644 --- a/matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m +++ b/matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m @@ -7,10 +7,6 @@ addpath('./helpers'); find_Bosphorus; out_loc = './out_bosph/'; -if(~exist(out_loc, 'dir')) - mkdir(out_loc); -end - %% executable = '"../../x64/Release/FaceLandmarkImg.exe"'; @@ -22,8 +18,8 @@ parfor f1=1:numel(bosph_dirs) command = executable; input_dir = [Bosphorus_dir, '/BosphorusDB/BosphorusDB/', bosph_dirs(f1).name]; - command = cat(2, command, [' -fdir "' input_dir '" -ofdir "' out_loc '"']); - command = cat(2, command, ' -multi_view 1 -wild -q'); + command = cat(2, command, [' -fdir "' input_dir '" -out_dir "' out_loc '"']); + command = cat(2, command, ' -multi_view 1 -wild -aus '); dos(command); @@ -37,64 +33,32 @@ aus_Bosph = [1, 2, 4, 5, 6, 7, 9, 10, 12, 14, 15, 17, 20, 23, 25, 26, 45]; %% Read the predicted values -% First read the first file to get the ids and line numbers -% au occurences -fid = fopen([out_loc, filenames{1}, '_det_0.pts']); -data = fgetl(fid); - -ind = 0; -beg_ind = -1; -end_ind = -1; -aus_det = []; -aus_det_id = []; - -%% -while ischar(data) - if(~isempty(findstr(data, 'au occurences:'))) - num_occurences = str2num(data(numel('au occurences:')+1:end)); - % Skip ahead two lines - data = fgetl(fid); - data = fgetl(fid); - ind = ind + 2; - beg_ind = ind; - end - - if(beg_ind ~= -1 && end_ind == -1) - if(~isempty(findstr(data, '}'))) - end_ind = ind; - else - d = strsplit(data, ' '); - aus_det = cat(1, aus_det, str2num(d{1}(3:end))); - aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1); - end - end - - data = fgetl(fid); - ind = ind + 1; +% First read the first file to get the column ids +tab = readtable([out_loc, filenames{1}, '.csv']); +column_names = tab.Properties.VariableNames; +aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_c')); +aus_det_cell = column_names(aus_det_id); +aus_det = zeros(size(aus_det_cell)); +for i=1:numel(aus_det) + aus_det(i) = str2num(aus_det_cell{i}(3:4)); end -fclose(fid); %% labels_pred = zeros(size(labels_gt)); for i=1:numel(filenames) % Will need to read the relevant AUs only - if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file')) - fid = fopen([out_loc, filenames{i}, '_det_0.pts']); - for k=1:beg_ind - data = fgetl(fid); + all_params = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0); + + % if multiple faces detected just take the first row + aus_pred = all_params(1, aus_det_id); + + for k=1:numel(aus_det) + if(sum(aus_Bosph == aus_det(k))>0) + labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k); end - - for k=1:num_occurences - data = fgetl(fid); - if(sum(aus_Bosph == aus_det(k))>0) - d = strsplit(data, ' '); - labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2}); - end - end - - fclose(fid); end + end %% @@ -122,63 +86,32 @@ fclose(f); %% Read the predicted values for intensities -% First read the first file to get the ids and line numbers -% au occurences -fid = fopen([out_loc, filenames{1}, '_det_0.pts']); -data = fgetl(fid); - -ind = 0; -beg_ind = -1; -end_ind = -1; -aus_det = []; -aus_det_id = []; - -while ischar(data) - if(~isempty(findstr(data, 'au intensities:'))) - num_occurences = str2num(data(numel('au intensities:')+1:end)); - % Skip ahead two lines - data = fgetl(fid); - data = fgetl(fid); - ind = ind + 2; - beg_ind = ind; - end - - if(beg_ind ~= -1 && end_ind == -1) - if(~isempty(findstr(data, '}'))) - end_ind = ind; - else - d = strsplit(data, ' '); - aus_det = cat(1, aus_det, str2num(d{1}(3:end))); - aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1); - end - end - - data = fgetl(fid); - ind = ind + 1; +% First read the first file to get the column ids +tab = readtable([out_loc, filenames{1}, '.csv']); +column_names = tab.Properties.VariableNames; +aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r')); +aus_det_cell = column_names(aus_det_id); +aus_det = zeros(size(aus_det_cell)); +for i=1:numel(aus_det) + aus_det(i) = str2num(aus_det_cell{i}(3:4)); end -fclose(fid); %% labels_pred = zeros(size(labels_gt)); for i=1:numel(filenames) % Will need to read the relevant AUs only - if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file')) - fid = fopen([out_loc, filenames{i}, '_det_0.pts']); - for k=1:beg_ind - data = fgetl(fid); + all_params = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0); + + % if multiple faces detected just take the first row + aus_pred = all_params(1, aus_det_id); + + for k=1:numel(aus_det) + if(sum(aus_Bosph == aus_det(k))>0) + labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k); end - - for k=1:num_occurences - data = fgetl(fid); - if(sum(aus_Bosph == aus_det(k))>0) - d = strsplit(data, ' '); - labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2}); - end - end - - fclose(fid); end + end %% diff --git a/matlab_runners/Full_test_suite.m b/matlab_runners/Full_test_suite.m index d91f631..e52d470 100644 --- a/matlab_runners/Full_test_suite.m +++ b/matlab_runners/Full_test_suite.m @@ -23,7 +23,7 @@ cd('../'); cd('Action Unit Experiments'); run_AU_prediction_Bosphorus assert(mean(cccs_reg) > 0.56); -assert(mean(f1s_class) > 0.46); +assert(mean(f1s_class) > 0.49); run_AU_prediction_BP4D assert(mean(ints_cccs) > 0.6);