Merge branch 'master' of https://github.com/TadasBaltrusaitis/OpenFace

Conflicts:
	.gitignore

.gitignore

@@ -20,9 +20,11 @@ matlab_runners/Feature Point Experiments/yt_features/
 matlab_runners/Feature Point Experiments/yt_features_clm/
 matlab_runners/Gaze Experiments/mpii_out/
 build/
+Release/AU_predictors/
 Release/
+exe/Recording/recording/
+ipch/
+exe/FeatureExtraction/out_bp4d/
 matlab_runners/Action Unit Experiments/out_bp4d/
 matlab_runners/Action Unit Experiments/out_SEMAINE/
-*.ipch
-exe/FeatureExtraction/out_bp4d/
 x64/Debug/

exe/FaceLandmarkImg/CMakeLists.txt

@@ -1,3 +1,6 @@
+#TBB library
+include_directories(${TBB_ROOT_DIR}/include)
+
 # Local libraries
 include_directories(${LandmarkDetector_SOURCE_DIR}/include)
 	

exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj

@@ -123,7 +123,7 @@
       <Optimization>Disabled</Optimization>
       <PreprocessorDefinitions>WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -159,7 +159,7 @@
       <PreprocessorDefinitions>WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <OpenMPSupport>false</OpenMPSupport>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

exe/FaceLandmarkVid/CMakeLists.txt

@@ -1,3 +1,6 @@
+#TBB library
+include_directories(${TBB_ROOT_DIR}/include)
+
 add_executable(FaceLandmarkVid FaceLandmarkVid.cpp)
 
 # Local libraries

exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj

@@ -125,7 +125,7 @@
       <PreprocessorDefinitions>WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <OpenMPSupport>false</OpenMPSupport>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -165,7 +165,7 @@
       <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <OpenMPSupport>false</OpenMPSupport>
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

exe/FaceLandmarkVidMulti/CMakeLists.txt

@@ -1,3 +1,6 @@
+#TBB library
+include_directories(${TBB_ROOT_DIR}/include)
+
 # Local libraries
 include_directories(${LandmarkDetector_SOURCE_DIR}/include)
 	

exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj

@@ -113,7 +113,7 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
       <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
@@ -142,7 +142,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
     </ClCompile>
     <Link>

exe/FeatureExtraction/CMakeLists.txt

@@ -1,3 +1,6 @@
+#TBB library
+include_directories(${TBB_ROOT_DIR}/include)
+
 add_executable(FeatureExtraction FeatureExtraction.cpp)
 
 # Local libraries

exe/FeatureExtraction/FeatureExtraction.vcxproj

@@ -124,7 +124,7 @@
       <PreprocessorDefinitions>WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <OpenMPSupport>false</OpenMPSupport>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -163,7 +163,7 @@
       <AdditionalIncludeDirectories>$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <OpenMPSupport>false</OpenMPSupport>
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
     </ClCompile>
     <Link>

lib/3rdParty/dlib/dlib.vcxproj

@@ -118,7 +118,7 @@
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <CompileAs>CompileAsCpp</CompileAs>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
       <ExceptionHandling>Sync</ExceptionHandling>
       <InlineFunctionExpansion>Disabled</InlineFunctionExpansion>
       <Optimization>Disabled</Optimization>
@@ -175,7 +175,7 @@
     <ClCompile>
       <AdditionalIncludeDirectories>$(SolutionDir)dlib/include/dlib/..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <CompileAs>CompileAsCpp</CompileAs>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
       <ExceptionHandling>Sync</ExceptionHandling>
       <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
       <Optimization>Full</Optimization>

lib/local/FaceAnalyser/CMakeLists.txt

@@ -1,3 +1,6 @@
+#TBB library
+include_directories(${TBB_ROOT_DIR}/include)
+
 include_directories(${BOOST_INCLUDE_DIR})
 
 SET(SOURCE

lib/local/FaceAnalyser/FaceAnalyser.vcxproj

@@ -113,7 +113,7 @@
       <Optimization>Disabled</Optimization>
       <SDLCheck>false</SDLCheck>
       <AdditionalIncludeDirectories>./include;$(SolutionDir)lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
@@ -158,7 +158,7 @@
       <SDLCheck>
       </SDLCheck>
       <AdditionalIncludeDirectories>./include;$(SolutionDir)lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>

lib/local/LandmarkDetector/CMakeLists.txt

@@ -1,3 +1,6 @@
+#TBB library
+include_directories(${TBB_ROOT_DIR}/include)
+
 include_directories(${BOOST_INCLUDE_DIR})
 
 SET(SOURCE

lib/local/LandmarkDetector/LandmarkDetector.vcxproj

@@ -119,7 +119,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
       <WarningLevel>Level3</WarningLevel>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
       <OpenMPSupport>false</OpenMPSupport>
       <AdditionalOptions>/Zm300 %(AdditionalOptions)</AdditionalOptions>
     </ClCompile>
@@ -166,7 +166,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c
       <PrecompiledHeader>Use</PrecompiledHeader>
       <WarningLevel>Level3</WarningLevel>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
       <OpenMPSupport>false</OpenMPSupport>
       <AdditionalOptions>/Zm300 %(AdditionalOptions)</AdditionalOptions>

matlab_version/demo/face_image_demo_eyes.m

@@ -9,22 +9,30 @@ addpath('../CCNF/');
    
 [clmParams, pdm] = Load_CLM_params_wild();
 
-[clmParams_eye, pdm_eye] = Load_CLM_params_eye();
-
 % An accurate CCNF (or CLNF) model
 % [patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*_general.mat', [], [], clmParams);
 % A simpler (but less accurate SVR)
 [patches] = Load_Patch_Experts( '../models/general/', 'svr_patches_*_general.mat', [], [], clmParams);
 
-[patches_eye] = Load_Patch_Experts( 'C:\Users\Tadas\Dropbox\AAM\patch_experts_eyes\svr_training\trained/', 'svr_patches_*_synth.mat', [], [], clmParams);
+% Loading eye PDM and patch experts
+[clmParams_eye, pdm_right_eye, pdm_left_eye] = Load_CLM_params_eye_28();
+[patches_right_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_right_eye.mat', [], [], clmParams_eye);
+[patches_left_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_left_eye.mat', [], [], clmParams_eye);
+clmParams_eye.multi_modal_types  = patches_right_eye(1).multi_modal_types;
+right_eye_inds = [43,44,45,46,47,48];
+left_eye_inds = [37,38,39,40,41,42];
+
+right_eye_inds_synth = [9 11 13 15 17 19];
+left_eye_inds_synth = [9 11 13 15 17 19];
 
 clmParams.multi_modal_types  = patches(1).multi_modal_types;
 
-clmParams_eye.multi_modal_types  = patches_eye(1).multi_modal_types;
-
 %%
-root_dir = 'C:\Users\Tadas\Dropbox\AAM\test data\gaze_original\p00/';
+% root_dir = 'C:\Users\Tadas\Dropbox\AAM\test data\gaze_original\p00/';
-images = dir([root_dir, '*.jpg']);
+% images = dir([root_dir, '*.jpg']);
+
+root_dir = './sample_eye_imgs/';
+images = dir([root_dir, '/*.png']);
 
 verbose = true;
 
@@ -84,42 +92,35 @@ for img=1:numel(images)
         % shape correction for matlab format
         shape = shape + 1;
               
-        if(verbose)
+        % Perform eye fitting now
+        shape_r_eye = zeros(numel(pdm_right_eye.M)/3, 2);
+        shape_r_eye(right_eye_inds_synth,:) = shape(right_eye_inds, :);
 
-            % valid points to draw (not to draw self-occluded ones)
+        [ a, R, T, ~, l_params] = fit_PDM_ortho_proj_to_2D(pdm_right_eye.M, pdm_right_eye.E, pdm_right_eye.V, shape_r_eye);
-            v_points = logical(patches(1).visibilities(view_used,:));
 
-            try
+        bbox = [min(shape_r_eye(:,1)), min(shape_r_eye(:,2)), max(shape_r_eye(:,1)), max(shape_r_eye(:,2))];
-
-            plot(shape(v_points,1), shape(v_points',2),'.r','MarkerSize',20);
-            plot(shape(v_points,1), shape(v_points',2),'.b','MarkerSize',10);
-
-            catch warn
-
-            end
-        end
-        
-        % Map from detected landmarks to eye params
-        shape_r_eye = zeros(20,2);
-        shape_r_eye([9,11,13,15,17,19],:) = shape([43,44,45,46,47,48], :);
-        
-        [ a, R, T, ~, params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm_eye.M, pdm_eye.E, pdm_eye.V, shape_r_eye);
 
         g_param = [a; Rot2Euler(R)'; T];
-        l_param = params;
 
-        % Use the initial global and local params for clm fitting in the image
+        [shape_r_eye] = Fitting_from_bb(image, [], bbox, pdm_right_eye, patches_right_eye, clmParams_eye, 'gparam', g_param, 'lparam', l_params);
-        patches_eye(1).visibilities(1:8) = 0;
-        patches_eye(2).visibilities(1:8) = 0;
-        patches_eye(3).visibilities(1:8) = 0;
-        [shape_eye,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm_eye, patches_eye, clmParams_eye, 'gparam', g_param, 'lparam', l_param);
 
-        plot(shape_eye(:,1), shape_eye(:,2), '.g', 'MarkerSize',15);
+        % Perform eye fitting now 
-%         % Now do the eyes
+        shape_l_eye = zeros(numel(pdm_right_eye.M)/3, 2);        
-%         min_x = shape(43,1);
+        shape_l_eye(left_eye_inds_synth,:) = shape(left_eye_inds, :);
-%         max_x = shape(43,1);
-%         bbox_eye = shape(43,1)
 
+        [ a, R, T, ~, l_params] = fit_PDM_ortho_proj_to_2D(pdm_left_eye.M, pdm_left_eye.E, pdm_left_eye.V, shape_l_eye);
+
+        bbox = [min(shape_l_eye(:,1)), min(shape_l_eye(:,2)), max(shape_l_eye(:,1)), max(shape_l_eye(:,2))];
+
+        g_param = [a; Rot2Euler(R)'; T];
+
+        [shape_l_eye] = Fitting_from_bb(image, [], bbox, pdm_left_eye, patches_left_eye, clmParams_eye, 'gparam', g_param, 'lparam', l_params);
+
+        plot(shape_l_eye(9:20,1), shape_l_eye(9:20,2), '.g', 'MarkerSize',15);
+        plot(shape_l_eye(1:8,1), shape_l_eye(1:8,2), '.b', 'MarkerSize',15);
+
+        plot(shape_r_eye(9:20,1), shape_r_eye(9:20,2), '.g', 'MarkerSize',15);
+        plot(shape_r_eye(1:8,1), shape_r_eye(1:8,2), '.b', 'MarkerSize',15);
     end
     hold off;
     

matlab_version/experiments_in_the_wild/Script_PDM_eyes.m

@@ -1,80 +0,0 @@
-function Script_PDM_eyes()
-
-addpath('../PDM_helpers/');
-addpath('../fitting/normxcorr2_mex_ALL');
-addpath('../fitting/');
-addpath('../CCNF/');
-addpath('../models/');
-
-% Replace this with the location of in 300 faces in the wild data
-if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file'))
-    root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/'];    
-else
-    root_test_data = 'F:/Dropbox/Dropbox/AAM/test data/';
-end
-
-[images, detections, labels] = Collect_wild_imgs(root_test_data);
-
-%% Fitting the model to the provided image
-
-% the default PDM to use
-pdmLoc = ['../models/pdm/pdm_68_aligned_wild_eyes.mat'];
-
-load(pdmLoc);
-
-pdm = struct;
-pdm.M = double(M);
-pdm.E = double(E);
-pdm.V = double(V);
-
-num_points = numel(M)/3;
-
-errors = zeros(numel(images),1);
-shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
-labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
-errors_normed = zeros(numel(images),1);
-
-errors_left_eye = zeros(numel(images),1);
-errors_right_eye = zeros(numel(images),1);
-
-tic
-for i=1:numel(images)
-
-    image = imread(images(i).img);
-    image_orig = image;
-    
-    if(size(image,3) == 3)
-        image = rgb2gray(image);
-    end              
-    
-    labels_curr = squeeze(labels(i,:,:));
-    
-    [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, labels_curr);
-
-    shape = shapeOrtho;
-    shapes_all(:,:,i) = shapeOrtho;
-    labels_all(:,:,i) = labels_curr;
-
-    if(mod(i, 200)==0)
-        fprintf('%d done\n', i );
-    end
-
-    valid_points =  sum(squeeze(labels(i,:,:)),2) > 0;
-    valid_points(1:17) = 0;
-
-    actualShape = squeeze(labels(i,:,:));
-    errors(i) = sqrt(mean(sum((actualShape(valid_points,:) - shape(valid_points,:)).^2,2)));      
-    width = ((max(actualShape(valid_points,1)) - min(actualShape(valid_points,1)))+(max(actualShape(valid_points,2)) - min(actualShape(valid_points,2))))/2;
-    errors_normed(i) = errors(i)/width;     
-
-    errors_left_eye(i) = compute_error_point_to_line_left_eye(actualShape, shapeOrtho, [0]);
-    errors_right_eye(i) = compute_error_point_to_line_right_eye(actualShape, shapeOrtho, [0]);
-
-    if(errors_normed(i) > 0.035 || errors_left_eye(i) > 0.035 || errors_right_eye(i) > 0.035)
-        imshow(image);hold on; plot(shape(:,1), shape(:,2), '.g'); hold off;
-    end
-end
-  
-save('Errors_PDM_eyes.mat', 'errors_normed', 'errors_left_eye', 'errors_right_eye');
-
-end

matlab_version/experiments_in_the_wild/Script_PDM_general.m

@@ -1,80 +0,0 @@
-function Script_PDM_general()
-
-addpath('../PDM_helpers/');
-addpath('../fitting/normxcorr2_mex_ALL');
-addpath('../fitting/');
-addpath('../CCNF/');
-addpath('../models/');
-
-% Replace this with the location of in 300 faces in the wild data
-if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file'))
-    root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/'];    
-else
-    root_test_data = 'F:/Dropbox/Dropbox/AAM/test data/';
-end
-
-[images, detections, labels] = Collect_wild_imgs(root_test_data);
-
-%% Fitting the model to the provided image
-
-% the default PDM to use
-pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
-
-load(pdmLoc);
-
-pdm = struct;
-pdm.M = double(M);
-pdm.E = double(E);
-pdm.V = double(V);
-
-num_points = numel(M)/3;
-
-errors = zeros(numel(images),1);
-shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
-labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
-errors_normed = zeros(numel(images),1);
-
-errors_left_eye = zeros(numel(images),1);
-errors_right_eye = zeros(numel(images),1);
-
-tic
-for i=1:numel(images)
-
-    image = imread(images(i).img);
-    image_orig = image;
-    
-    if(size(image,3) == 3)
-        image = rgb2gray(image);
-    end              
-    
-    labels_curr = squeeze(labels(i,:,:));
-    
-    [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, labels_curr);
-
-    shape = shapeOrtho;
-    shapes_all(:,:,i) = shapeOrtho;
-    labels_all(:,:,i) = labels_curr;
-
-    if(mod(i, 200)==0)
-        fprintf('%d done\n', i );
-    end
-
-    valid_points =  sum(squeeze(labels(i,:,:)),2) > 0;
-    valid_points(1:17) = 0;
-
-    actualShape = squeeze(labels(i,:,:));
-    errors(i) = sqrt(mean(sum((actualShape(valid_points,:) - shape(valid_points,:)).^2,2)));      
-    width = ((max(actualShape(valid_points,1)) - min(actualShape(valid_points,1)))+(max(actualShape(valid_points,2)) - min(actualShape(valid_points,2))))/2;
-    errors_normed(i) = errors(i)/width;     
-
-    errors_left_eye(i) = compute_error_point_to_line_left_eye(actualShape, shapeOrtho, [0]);
-    errors_right_eye(i) = compute_error_point_to_line_right_eye(actualShape, shapeOrtho, [0]);
-
-    if(errors_normed(i) > 0.035 || errors_left_eye(i) > 0.035 || errors_right_eye(i) > 0.035)
-        imshow(image);hold on; plot(shape(:,1), shape(:,2), '.g'); hold off;
-    end
-end
-  
-save('Errors_PDM_basic.mat', 'errors_normed', 'errors_left_eye', 'errors_right_eye');
-
-end

matlab_version/experiments_in_the_wild/compute_error_point_to_line_left_eye.m

@@ -1,86 +0,0 @@
-function [ error_per_image ] = compute_error_point_to_line_left_eye( ground_truth_all, detected_points_all, occluded )
-%compute_error
-%   compute the average point-to-point Euclidean error normalized by the
-%   inter-ocular distance (measured as the Euclidean distance between the
-%   outer corners of the eyes)
-%
-%   Inputs:
-%          grounth_truth_all, size: num_of_points x 2 x num_of_images
-%          detected_points_all, size: num_of_points x 2 x num_of_images
-%   Output:
-%          error_per_image, size: num_of_images x 1
-
-
-
-right_eye_inds_from_68 = [37,38,39,40,41,42,37];
-right_eye_inds_from_28 = [9,11,13,15,17,19];
-
-num_of_images = size(ground_truth_all,3);
-
-num_points_gt = size(ground_truth_all,1);
-
-num_points_det = size(detected_points_all,1);
-
-error_per_image = zeros(num_of_images,1);
-
-for i =1:num_of_images
-    
-    if(num_points_det == 6)
-        detected_points = detected_points_all(:,:,i);
-    elseif(num_points_det == 68 || num_points_det == 66)
-        detected_points = detected_points_all(right_eye_inds_from_68,:,i);        
-    elseif(num_points_det == 28)
-        detected_points = detected_points_all(right_eye_inds_from_28,:,i);        
-    elseif(num_points_det == 49)
-        
-    end
-    ground_truth_points  = ground_truth_all(:,:,i);
-    
-    if(num_points_gt == 66 || num_points_gt == 68)
-        interocular_distance = norm(ground_truth_points(37,:)-ground_truth_points(46,:));
-        ground_truth_points = ground_truth_points(right_eye_inds_from_68,:,:);
-    else
-        interocular_distance = norm(ground_truth_points(37-17,:)-ground_truth_points(46-17,:));
-        ground_truth_points = ground_truth_points(right_eye_inds_from_68,:,:);
-    end
-    
-    sum=0;
-    for j=1:6
-        
-        if(j== 1 || j == 6)
-            % eye corners should align perfectly
-            sum = sum + norm(detected_points(j,:)-ground_truth_points(j,:));
-        else
-            % points between eye corners measured in distance to the two appropriate line
-            % segments
-            sum = sum + point_to_segments(detected_points(j,:), ground_truth_points(j-1:j+1,:));
-        end
-    end
-    error_per_image(i) = sum/(6*interocular_distance);
-end
-
-error_per_image = error_per_image(~occluded);
-
-end
-
-function seg_dist = point_to_segments(point, segments)
-    
-    seg_dists = zeros(size(segments, 1)-1,1);
-
-    for i=1:size(segments, 1)-1
-       
-        vec1 = point - segments(i,:);
-        vec2 = segments(i+1,:) - segments(i,:);
-        
-        d = (vec1 * vec2') / (norm(vec2)^2);
-        
-        if(d < 0)
-            seg_dists(i) = norm(vec1);
-        elseif(d > 1)
-            seg_dists(i) = norm(point - segments(i+1,:));
-        else
-            seg_dists(i) = sqrt( norm(vec1)^2 - norm(d * vec2)^2);
-        end
-    end
-    seg_dist = min(seg_dists);
-end

matlab_version/experiments_in_the_wild/compute_error_point_to_line_right_eye.m

@@ -1,86 +0,0 @@
-function [ error_per_image ] = compute_error_point_to_line_right_eye( ground_truth_all, detected_points_all, occluded )
-%compute_error
-%   compute the average point-to-point Euclidean error normalized by the
-%   inter-ocular distance (measured as the Euclidean distance between the
-%   outer corners of the eyes)
-%
-%   Inputs:
-%          grounth_truth_all, size: num_of_points x 2 x num_of_images
-%          detected_points_all, size: num_of_points x 2 x num_of_images
-%   Output:
-%          error_per_image, size: num_of_images x 1
-
-
-
-right_eye_inds_from_68 = [43,44,45,46,47,48,43];
-right_eye_inds_from_28 = [9,11,13,15,17,19];
-
-num_of_images = size(ground_truth_all,3);
-
-num_points_gt = size(ground_truth_all,1);
-
-num_points_det = size(detected_points_all,1);
-
-error_per_image = zeros(num_of_images,1);
-
-for i =1:num_of_images
-    
-    if(num_points_det == 6)
-        detected_points = detected_points_all(:,:,i);
-    elseif(num_points_det == 68 || num_points_det == 66)
-        detected_points = detected_points_all(right_eye_inds_from_68,:,i);        
-    elseif(num_points_det == 28)
-        detected_points = detected_points_all(right_eye_inds_from_28,:,i);        
-    elseif(num_points_det == 49)
-        
-    end
-    ground_truth_points  = ground_truth_all(:,:,i);
-    
-    if(num_points_gt == 66 || num_points_gt == 68)
-        interocular_distance = norm(ground_truth_points(37,:)-ground_truth_points(46,:));
-        ground_truth_points = ground_truth_points(right_eye_inds_from_68,:,:);
-    else
-        interocular_distance = norm(ground_truth_points(37-17,:)-ground_truth_points(46-17,:));
-        ground_truth_points = ground_truth_points(right_eye_inds_from_68,:,:);
-    end
-    
-    sum=0;
-    for j=1:6
-        
-        if(j== 1 || j == 6)
-            % eye corners should align perfectly
-            sum = sum + norm(detected_points(j,:)-ground_truth_points(j,:));
-        else
-            % points between eye corners measured in distance to the two appropriate line
-            % segments
-            sum = sum + point_to_segments(detected_points(j,:), ground_truth_points(j-1:j+1,:));
-        end
-    end
-    error_per_image(i) = sum/(6*interocular_distance);
-end
-
-error_per_image = error_per_image(~occluded);
-
-end
-
-function seg_dist = point_to_segments(point, segments)
-    
-    seg_dists = zeros(size(segments, 1)-1,1);
-
-    for i=1:size(segments, 1)-1
-       
-        vec1 = point - segments(i,:);
-        vec2 = segments(i+1,:) - segments(i,:);
-        
-        d = (vec1 * vec2') / (norm(vec2)^2);
-        
-        if(d < 0)
-            seg_dists(i) = norm(vec1);
-        elseif(d > 1)
-            seg_dists(i) = norm(point - segments(i+1,:));
-        else
-            seg_dists(i) = sqrt( norm(vec1)^2 - norm(d * vec2)^2);
-        end
-    end
-    seg_dist = min(seg_dists);
-end

matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_eyes.m

@@ -27,8 +27,12 @@ pdm.M = double(M);
 pdm.E = double(E);
 pdm.V = double(V);
 
+num_points = numel(M)/3;
+
+errors = zeros(numel(images),1);
 shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
 labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+errors_normed = zeros(numel(images),1);
 
 errors_left_eye = zeros(numel(images),1);
 errors_right_eye = zeros(numel(images),1);
@@ -36,30 +40,41 @@ errors_right_eye = zeros(numel(images),1);
 tic
 for i=1:numel(images)
 
+    image = imread(images(i).img);
+    image_orig = image;
+    
+    if(size(image,3) == 3)
+        image = rgb2gray(image);
+    end              
+    
     labels_curr = squeeze(labels(i,:,:));
     
-    [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D_no_reg(pdm.M, pdm.E, pdm.V, labels_curr);
+    [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, labels_curr);
 
+    shape = shapeOrtho;
     shapes_all(:,:,i) = shapeOrtho;
     labels_all(:,:,i) = labels_curr;
 
-    if(mod(i, 100)==0)
+    if(mod(i, 200)==0)
         fprintf('%d done\n', i );
     end
 
+    valid_points =  sum(squeeze(labels(i,:,:)),2) > 0;
+    valid_points(1:17) = 0;
+
     actualShape = squeeze(labels(i,:,:));
+    errors(i) = sqrt(mean(sum((actualShape(valid_points,:) - shape(valid_points,:)).^2,2)));      
+    width = ((max(actualShape(valid_points,1)) - min(actualShape(valid_points,1)))+(max(actualShape(valid_points,2)) - min(actualShape(valid_points,2))))/2;
+    errors_normed(i) = errors(i)/width;     
 
     errors_left_eye(i) = compute_error_point_to_line_left_eye(actualShape, shapeOrtho, [0]);
     errors_right_eye(i) = compute_error_point_to_line_right_eye(actualShape, shapeOrtho, [0]);
 
-    if(errors_left_eye(i) > 0.02 || errors_right_eye(i) > 0.02)
+    if(errors_normed(i) > 0.035 || errors_left_eye(i) > 0.035 || errors_right_eye(i) > 0.035)
-        plot(shapeOrtho(:,1), -shapeOrtho(:,2), 'r.'); hold on;
+        imshow(image);hold on; plot(shape(:,1), shape(:,2), '.g'); hold off;
-        axis equal;
-        plot(labels_curr(:,1), -labels_curr(:,2), 'g.'); hold off;
     end
-
 end
   
-save('Errors_PDM_eyes.mat', 'errors_left_eye', 'errors_right_eye');
+save('Errors_PDM_eyes.mat', 'errors_normed', 'errors_left_eye', 'errors_right_eye');
 
 end

matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_general.m

@@ -1,4 +1,4 @@
-function Script_PDM_eyes()
+function Script_PDM_general()
 
 addpath('../PDM_helpers/');
 addpath('../fitting/normxcorr2_mex_ALL');
@@ -27,8 +27,12 @@ pdm.M = double(M);
 pdm.E = double(E);
 pdm.V = double(V);
 
+num_points = numel(M)/3;
+
+errors = zeros(numel(images),1);
 shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
 labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+errors_normed = zeros(numel(images),1);
 
 errors_left_eye = zeros(numel(images),1);
 errors_right_eye = zeros(numel(images),1);
@@ -36,31 +40,41 @@ errors_right_eye = zeros(numel(images),1);
 tic
 for i=1:numel(images)
 
+    image = imread(images(i).img);
+    image_orig = image;
+    
+    if(size(image,3) == 3)
+        image = rgb2gray(image);
+    end              
+    
     labels_curr = squeeze(labels(i,:,:));
     
-    [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D_no_reg(pdm.M, pdm.E, pdm.V, labels_curr);
+    [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, labels_curr);
 
+    shape = shapeOrtho;
     shapes_all(:,:,i) = shapeOrtho;
     labels_all(:,:,i) = labels_curr;
 
-    if(mod(i, 100)==0)
+    if(mod(i, 200)==0)
         fprintf('%d done\n', i );
     end
 
+    valid_points =  sum(squeeze(labels(i,:,:)),2) > 0;
+    valid_points(1:17) = 0;
+
     actualShape = squeeze(labels(i,:,:));
+    errors(i) = sqrt(mean(sum((actualShape(valid_points,:) - shape(valid_points,:)).^2,2)));      
+    width = ((max(actualShape(valid_points,1)) - min(actualShape(valid_points,1)))+(max(actualShape(valid_points,2)) - min(actualShape(valid_points,2))))/2;
+    errors_normed(i) = errors(i)/width;     
 
     errors_left_eye(i) = compute_error_point_to_line_left_eye(actualShape, shapeOrtho, [0]);
     errors_right_eye(i) = compute_error_point_to_line_right_eye(actualShape, shapeOrtho, [0]);
 
-    
+    if(errors_normed(i) > 0.035 || errors_left_eye(i) > 0.035 || errors_right_eye(i) > 0.035)
-    if(errors_left_eye(i) > 0.02 || errors_right_eye(i) > 0.02)
+        imshow(image);hold on; plot(shape(:,1), shape(:,2), '.g'); hold off;
-        plot(shapeOrtho(:,1), -shapeOrtho(:,2), 'r.'); hold on;
-        axis equal;
-        plot(labels_curr(:,1), -labels_curr(:,2), 'g.'); hold off;
     end
-    
 end
   
-save('Errors_PDM_basic.mat', 'errors_left_eye', 'errors_right_eye');
+save('Errors_PDM_basic.mat', 'errors_normed', 'errors_left_eye', 'errors_right_eye');
 
 end

matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_left_eye.m

@@ -59,9 +59,7 @@ for i =1:num_of_images
     error_per_image(i) = sum/(6*interocular_distance);
 end
 
-if(nargin > 2)
+error_per_image = error_per_image(~occluded);
-    error_per_image = error_per_image(~occluded);
-end
 
 end
 

matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_right_eye.m

@@ -59,9 +59,7 @@ for i =1:num_of_images
     error_per_image(i) = sum/(6*interocular_distance);
 end
 
-if(nargin > 2)
+error_per_image = error_per_image(~occluded);
-    error_per_image = error_per_image(~occluded);
-end
 
 end
 

matlab_version/models/Load_CLM_params_eye_28.m

@@ -0,0 +1,38 @@
+function [ clmParams, pdm_right, pdm_left ] = Load_CLM_params_eye_28() 
+%LOAD_CLM_PARAMS_WILD Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams.window_size = [17,17; 15,15; 13,13;];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+    
+    % the PDM created from in the wild data
+    pdmLoc = ['../models/hierarch_pdm/pdm_28_r_eye.mat'];
+
+    load(pdmLoc);
+
+    pdm_right = struct;
+    pdm_right.M = double(M);
+    pdm_right.E = double(E);
+    pdm_right.V = double(V);
+
+    pdmLoc = ['../models/hierarch_pdm/pdm_28_l_eye.mat'];
+
+    load(pdmLoc);
+
+    pdm_left = struct;
+    pdm_left.M = double(M);
+    pdm_left.E = double(E);
+    pdm_left.V = double(V);
+    
+    % the default model parameters to use
+    clmParams.regFactor = 2.0;               
+    clmParams.sigmaMeanShift = 1.5;
+    clmParams.tikhonov_factor = 0;
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 0;
+    clmParams.tikhonov_factor = 0;
+end
+