Bosphorus experiments with the new interface, improved the results slightly.

lib/local/LandmarkDetector/include/LandmarkDetectorModel.h

@@ -93,15 +93,12 @@ public:
 	dlib::frontal_face_detector face_detector_HOG;
 
 
-	// Validate if the detected landmarks are correct using an SVR regressor
+	// Validate if the detected landmarks are correct using a predictor on detected landmarks
 	DetectionValidator	landmark_validator; 
 
-	// Indicating if landmark detection succeeded (based on SVR validator)
+	// Indicating if landmark detection succeeded (based on detection validator)
 	bool				detection_success; 
 
-	// Indicating if the tracking has been initialised (for video based tracking)
-	bool				tracking_initialised;
-
 	//  Representing how confident we are that tracking succeeds (0 - complete failure, 1 - perfect success)
 	double				detection_certainty; 
 
@@ -174,8 +171,17 @@ public:
 	// Helper reading function
 	void Read_CLNF(string clnf_location);
 	
+	// Allows to set initialization accross hierarchical models as well
+	bool IsInitialized() const { return tracking_initialised; }
+	void SetInitialized(bool initialized);
+	void SetDetectionSuccess(bool detection_success);
+
 private:
 
+
+	// Indicating if the tracking has been initialised (for video based tracking)
+	bool				tracking_initialised;
+
 	// the speedup of RLMS using precalculated KDE responses (described in Saragih 2011 RLMS paper)
 	map<int, cv::Mat_<float> >		kde_resp_precalc;
 

lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp

@@ -215,10 +215,10 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
 	// and using a smaller search area
 
 	// Indicating that this is a first detection in video sequence or after restart
-	bool initial_detection = !clnf_model.tracking_initialised;
+	bool initial_detection = !clnf_model.IsInitialized();
 
 	// Only do it if there was a face detection at all
-	if(clnf_model.tracking_initialised)
+	if(clnf_model.IsInitialized())
 	{
 
 		// The area of interest search size will depend if the previous track was successful
@@ -254,8 +254,8 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
 
 	// This is used for both detection (if it the tracking has not been initialised yet) or if the tracking failed (however we do this every n frames, for speed)
 	// This also has the effect of an attempt to reinitialise just after the tracking has failed, which is useful during large motions
-	if((!clnf_model.tracking_initialised && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0) 
+	if((!clnf_model.IsInitialized() && (clnf_model.failures_in_a_row + 1) % (params.reinit_video_every * 6) == 0)
-		|| (clnf_model.tracking_initialised && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0))
+		|| (clnf_model.IsInitialized() && !clnf_model.detection_success && params.reinit_video_every > 0 && clnf_model.failures_in_a_row % params.reinit_video_every == 0))
 	{
 
 		cv::Rect_<double> bounding_box;
@@ -290,7 +290,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
 		if(face_detection_success)
 		{
 			// Indicate that tracking has started as a face was detected
-			clnf_model.tracking_initialised = true;
+			clnf_model.SetInitialized(true);
 						
 			// Keep track of old model values so that they can be restored if redetection fails
 			cv::Vec6d params_global_init = clnf_model.params_global;
@@ -335,7 +335,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
 	}
 
 	// if the model has not been initialised yet class it as a failure
-	if(!clnf_model.tracking_initialised)
+	if(!clnf_model.IsInitialized())
 	{
 		clnf_model.failures_in_a_row++;
 	}
@@ -343,7 +343,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
 	// un-initialise the tracking
 	if(	clnf_model.failures_in_a_row > 100)
 	{
-		clnf_model.tracking_initialised = false;
+		clnf_model.SetInitialized(false);
 	}
 
 	return clnf_model.detection_success;
@@ -359,7 +359,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_<uchar> &grayscale_i
 		clnf_model.pdm.CalcParams(clnf_model.params_global, bounding_box, clnf_model.params_local);		
 
 		// indicate that face was detected so initialisation is not necessary
-		clnf_model.tracking_initialised = true;
+		clnf_model.SetInitialized(true);
 	}
 
 	return DetectLandmarksInVideo(grayscale_image, clnf_model, params);
@@ -463,6 +463,9 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_<uchar> &grayscale_i
 		clnf_model.hierarchical_models[part].landmark_likelihoods = best_landmark_likelihoods_h[part].clone();
 	}
 
+	// To indicate that tracking/detection started and the values are valid, we assume that there is a face in the bounding box
+	clnf_model.SetInitialized(true);
+
 	return best_success;
 }
 

lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp

@@ -508,8 +508,8 @@ void CLNF::Read(string main_location)
 	detected_landmarks.create(2 * pdm.NumberOfPoints(), 1);
 	detected_landmarks.setTo(0);
 
-	detection_success = false;
+	SetDetectionSuccess(false);
-	tracking_initialised = false;
+	SetInitialized(false);
 	model_likelihood = -10; // very low
 	detection_certainty = 0; // very uncertain
 
@@ -526,13 +526,35 @@ void CLNF::Read(string main_location)
 
 }
 
+void CLNF::SetDetectionSuccess(bool success)
+{
+	this->detection_success = success;
+
+	for (size_t i = 0; i < hierarchical_models.size(); ++i) 
+	{
+		hierarchical_models[i].SetDetectionSuccess(success);
+	}
+
+}
+
+void CLNF::SetInitialized(bool initialized)
+{
+	this->tracking_initialised = initialized;
+
+	for (size_t i = 0; i < hierarchical_models.size(); ++i)
+	{
+		hierarchical_models[i].SetInitialized(initialized);
+	}
+
+}
+
 // Resetting the model (for a new video, or complet reinitialisation
 void CLNF::Reset()
 {
 	detected_landmarks.setTo(0);
 
-	detection_success = false;
+	SetDetectionSuccess(false);
-	tracking_initialised = false;
+	SetInitialized(false);
 	model_likelihood = -10;  // very low
 	detection_certainty = 0; // very uncertain
 
@@ -1084,7 +1106,7 @@ cv::Mat_<double> CLNF::GetShape(double fx, double fy, double cx, double cy) cons
 	cv::Mat_<double> outShape(n, 3, 0.0);
 
 	// If the tracking started (otherwise no point reporting 3D shape)
-	if(this->tracking_initialised)
+	if(this->IsInitialized())
 	{
 
 		cv::Mat_<double> shape3d(n * 3, 1);

matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt

@@ -1,17 +1,17 @@
-AU1 class, Precision - 0.393, Recall - 0.727, F1 - 0.510
+AU1 class, Precision - 0.434, Recall - 0.673, F1 - 0.528
-AU2 class, Precision - 0.266, Recall - 0.850, F1 - 0.405
+AU2 class, Precision - 0.298, Recall - 0.818, F1 - 0.437
-AU4 class, Precision - 0.511, Recall - 0.874, F1 - 0.645
+AU4 class, Precision - 0.564, Recall - 0.861, F1 - 0.681
-AU5 class, Precision - 0.294, Recall - 0.968, F1 - 0.451
+AU5 class, Precision - 0.387, Recall - 0.915, F1 - 0.544
-AU6 class, Precision - 0.346, Recall - 0.833, F1 - 0.489
+AU6 class, Precision - 0.355, Recall - 0.811, F1 - 0.494
-AU7 class, Precision - 0.793, Recall - 0.750, F1 - 0.771
+AU7 class, Precision - 0.778, Recall - 0.783, F1 - 0.780
-AU9 class, Precision - 0.316, Recall - 0.960, F1 - 0.475
+AU9 class, Precision - 0.370, Recall - 0.953, F1 - 0.533
-AU10 class, Precision - 0.349, Recall - 0.773, F1 - 0.481
+AU10 class, Precision - 0.340, Recall - 0.788, F1 - 0.475
-AU12 class, Precision - 0.674, Recall - 0.864, F1 - 0.757
+AU12 class, Precision - 0.690, Recall - 0.842, F1 - 0.758
-AU14 class, Precision - 0.183, Recall - 0.863, F1 - 0.302
+AU14 class, Precision - 0.185, Recall - 0.881, F1 - 0.305
-AU15 class, Precision - 0.183, Recall - 0.851, F1 - 0.302
+AU15 class, Precision - 0.171, Recall - 0.851, F1 - 0.285
-AU17 class, Precision - 0.293, Recall - 0.889, F1 - 0.441
+AU17 class, Precision - 0.309, Recall - 0.861, F1 - 0.455
-AU20 class, Precision - 0.114, Recall - 0.930, F1 - 0.203
+AU20 class, Precision - 0.130, Recall - 0.921, F1 - 0.228
-AU23 class, Precision - 0.107, Recall - 0.889, F1 - 0.191
+AU23 class, Precision - 0.104, Recall - 0.837, F1 - 0.186
-AU25 class, Precision - 0.860, Recall - 0.873, F1 - 0.866
+AU25 class, Precision - 0.869, Recall - 0.860, F1 - 0.865
-AU26 class, Precision - 0.359, Recall - 0.811, F1 - 0.498
+AU26 class, Precision - 0.368, Recall - 0.809, F1 - 0.506
-AU45 class, Precision - 0.318, Recall - 0.771, F1 - 0.450
+AU45 class, Precision - 0.367, Recall - 0.754, F1 - 0.494

matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt

@@ -1,17 +1,17 @@
-AU1 intensity, Corr - 0.717, RMS - 0.892, CCC - 0.668
+AU1 intensity, Corr - 0.712, RMS - 0.925, CCC - 0.652
-AU2 intensity, Corr - 0.696, RMS - 0.774, CCC - 0.625
+AU2 intensity, Corr - 0.696, RMS - 0.772, CCC - 0.626
-AU4 intensity, Corr - 0.802, RMS - 0.603, CCC - 0.776
+AU4 intensity, Corr - 0.797, RMS - 0.623, CCC - 0.764
-AU5 intensity, Corr - 0.747, RMS - 0.832, CCC - 0.640
+AU5 intensity, Corr - 0.767, RMS - 0.740, CCC - 0.694
-AU6 intensity, Corr - 0.556, RMS - 0.735, CCC - 0.533
+AU6 intensity, Corr - 0.541, RMS - 0.786, CCC - 0.506
-AU7 intensity, Corr - 0.831, RMS - 0.757, CCC - 0.804
+AU7 intensity, Corr - 0.830, RMS - 0.750, CCC - 0.811
-AU9 intensity, Corr - 0.779, RMS - 0.551, CCC - 0.738
+AU9 intensity, Corr - 0.763, RMS - 0.611, CCC - 0.701
-AU10 intensity, Corr - 0.495, RMS - 0.719, CCC - 0.475
+AU10 intensity, Corr - 0.491, RMS - 0.759, CCC - 0.461
-AU12 intensity, Corr - 0.810, RMS - 0.714, CCC - 0.753
+AU12 intensity, Corr - 0.804, RMS - 0.715, CCC - 0.766
-AU14 intensity, Corr - 0.348, RMS - 0.896, CCC - 0.280
+AU14 intensity, Corr - 0.357, RMS - 0.931, CCC - 0.277
-AU15 intensity, Corr - 0.527, RMS - 0.538, CCC - 0.448
+AU15 intensity, Corr - 0.516, RMS - 0.565, CCC - 0.431
-AU17 intensity, Corr - 0.561, RMS - 0.882, CCC - 0.484
+AU17 intensity, Corr - 0.554, RMS - 0.893, CCC - 0.477
-AU20 intensity, Corr - 0.413, RMS - 0.880, CCC - 0.285
+AU20 intensity, Corr - 0.411, RMS - 0.900, CCC - 0.277
-AU23 intensity, Corr - 0.354, RMS - 0.753, CCC - 0.268
+AU23 intensity, Corr - 0.351, RMS - 0.736, CCC - 0.274
-AU25 intensity, Corr - 0.847, RMS - 0.818, CCC - 0.811
+AU25 intensity, Corr - 0.846, RMS - 0.809, CCC - 0.822
-AU26 intensity, Corr - 0.514, RMS - 0.955, CCC - 0.465
+AU26 intensity, Corr - 0.516, RMS - 0.995, CCC - 0.453
-AU45 intensity, Corr - 0.868, RMS - 0.550, CCC - 0.848
+AU45 intensity, Corr - 0.840, RMS - 0.662, CCC - 0.791

matlab_runners/Action Unit Experiments/run_AU_prediction_Bosphorus.m

@@ -7,10 +7,6 @@ addpath('./helpers');
 find_Bosphorus;
 out_loc = './out_bosph/';
 
-if(~exist(out_loc, 'dir'))
-    mkdir(out_loc);
-end
-
 %%
 executable = '"../../x64/Release/FaceLandmarkImg.exe"';
 
@@ -22,8 +18,8 @@ parfor f1=1:numel(bosph_dirs)
     command = executable;
 
     input_dir = [Bosphorus_dir, '/BosphorusDB/BosphorusDB/', bosph_dirs(f1).name];
-    command = cat(2, command, [' -fdir "' input_dir '" -ofdir "' out_loc '"']);
+    command = cat(2, command, [' -fdir "' input_dir '" -out_dir "' out_loc '"']);
-    command = cat(2, command, ' -multi_view 1 -wild -q');
+    command = cat(2, command, ' -multi_view 1 -wild -aus ');
 
     dos(command);
 
@@ -37,64 +33,32 @@ aus_Bosph = [1, 2, 4, 5, 6, 7, 9, 10, 12, 14, 15, 17, 20, 23, 25, 26, 45];
 
 %% Read the predicted values
 
-% First read the first file to get the ids and line numbers
+% First read the first file to get the column ids
-% au occurences
+tab = readtable([out_loc, filenames{1}, '.csv']);
-fid = fopen([out_loc, filenames{1}, '_det_0.pts']);
+column_names = tab.Properties.VariableNames;
-data = fgetl(fid);
+aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_c'));
-
+aus_det_cell = column_names(aus_det_id);
-ind = 0;
+aus_det = zeros(size(aus_det_cell));
-beg_ind = -1;
+for i=1:numel(aus_det)
-end_ind = -1;
+    aus_det(i) = str2num(aus_det_cell{i}(3:4));
-aus_det = [];
-aus_det_id = [];
-
-%%
-while ischar(data)
-    if(~isempty(findstr(data, 'au occurences:')))
-        num_occurences = str2num(data(numel('au occurences:')+1:end));
-        % Skip ahead two lines
-        data = fgetl(fid);   
-        data = fgetl(fid);   
-        ind = ind + 2;
-        beg_ind = ind;
 end
 
-    if(beg_ind ~= -1 && end_ind == -1)
-        if(~isempty(findstr(data, '}')))
-            end_ind = ind;
-        else
-            d = strsplit(data, ' '); 
-            aus_det = cat(1, aus_det, str2num(d{1}(3:end)));
-            aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1);
-        end
-    end
-    
-    data = fgetl(fid);
-    ind = ind + 1;
-end
-fclose(fid);
-
 %%
 labels_pred = zeros(size(labels_gt));
 for i=1:numel(filenames)
 
     % Will need to read the relevant AUs only
-    if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file'))
+    all_params  = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0);
-        fid = fopen([out_loc, filenames{i}, '_det_0.pts']);
-        for k=1:beg_ind
-            data = fgetl(fid);
-        end
     
-        for k=1:num_occurences
+    % if multiple faces detected just take the first row
-            data = fgetl(fid);
+    aus_pred = all_params(1, aus_det_id);
+    
+    for k=1:numel(aus_det)
         if(sum(aus_Bosph == aus_det(k))>0)
-                d = strsplit(data, ' '); 
+            labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k);
-                labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2});
         end
     end
 
-        fclose(fid);
-    end
 end
 
 %%
@@ -122,63 +86,32 @@ fclose(f);
 
 %% Read the predicted values for intensities
 
-% First read the first file to get the ids and line numbers
+% First read the first file to get the column ids
-% au occurences
+tab = readtable([out_loc, filenames{1}, '.csv']);
-fid = fopen([out_loc, filenames{1}, '_det_0.pts']);
+column_names = tab.Properties.VariableNames;
-data = fgetl(fid);
+aus_det_id = cellfun(@(x) ~isempty(x) && x==5, strfind(column_names, '_r'));
-
+aus_det_cell = column_names(aus_det_id);
-ind = 0;
+aus_det = zeros(size(aus_det_cell));
-beg_ind = -1;
+for i=1:numel(aus_det)
-end_ind = -1;
+    aus_det(i) = str2num(aus_det_cell{i}(3:4));
-aus_det = [];
-aus_det_id = [];
-
-while ischar(data)
-    if(~isempty(findstr(data, 'au intensities:')))
-        num_occurences = str2num(data(numel('au intensities:')+1:end));
-        % Skip ahead two lines
-        data = fgetl(fid);   
-        data = fgetl(fid);   
-        ind = ind + 2;
-        beg_ind = ind;
 end
 
-    if(beg_ind ~= -1 && end_ind == -1)
-        if(~isempty(findstr(data, '}')))
-            end_ind = ind;
-        else
-            d = strsplit(data, ' '); 
-            aus_det = cat(1, aus_det, str2num(d{1}(3:end)));
-            aus_det_id = cat(1, aus_det_id, ind - beg_ind + 1);
-        end
-    end
-    
-    data = fgetl(fid);
-    ind = ind + 1;
-end
-fclose(fid);
-
 %%
 labels_pred = zeros(size(labels_gt));
 for i=1:numel(filenames)
 
     % Will need to read the relevant AUs only
-    if(exist([out_loc, filenames{i}, '_det_0.pts'], 'file'))
+    all_params  = dlmread([out_loc, filenames{i}, '.csv'], ',', 1, 0);
-        fid = fopen([out_loc, filenames{i}, '_det_0.pts']);
-        for k=1:beg_ind
-            data = fgetl(fid);
-        end
     
-        for k=1:num_occurences
+    % if multiple faces detected just take the first row
-            data = fgetl(fid);
+    aus_pred = all_params(1, aus_det_id);
+    
+    for k=1:numel(aus_det)
         if(sum(aus_Bosph == aus_det(k))>0)
-                d = strsplit(data, ' '); 
+            labels_pred(i, aus_Bosph == aus_det(k)) = aus_pred(k);
-                labels_pred(i, aus_Bosph == aus_det(k)) = str2num(d{2});
         end
     end
 
-        fclose(fid);
-    end
 end
 
 %%

matlab_runners/Full_test_suite.m

@@ -23,7 +23,7 @@ cd('../');
 cd('Action Unit Experiments');
 run_AU_prediction_Bosphorus
 assert(mean(cccs_reg) > 0.56);
-assert(mean(f1s_class) > 0.46);
+assert(mean(f1s_class) > 0.49);
 
 run_AU_prediction_BP4D
 assert(mean(ints_cccs) > 0.6);