diff --git a/.gitignore b/.gitignore index f5a4f42..e12c985 100644 --- a/.gitignore +++ b/.gitignore @@ -20,11 +20,13 @@ matlab_runners/Feature Point Experiments/yt_features/ matlab_runners/Feature Point Experiments/yt_features_clm/ matlab_runners/Gaze Experiments/mpii_out/ build/ +Release/AU_predictors/ Release/ +exe/Recording/recording/ +ipch/ +exe/FeatureExtraction/out_bp4d/ matlab_runners/Action Unit Experiments/out_bp4d/ matlab_runners/Action Unit Experiments/out_SEMAINE/ -*.ipch -exe/FeatureExtraction/out_bp4d/ x64/Debug/ matlab_runners/Action Unit Experiments/out_unbc/ matlab_runners/Action Unit Experiments/out_bosph/ diff --git a/exe/FaceLandmarkImg/CMakeLists.txt b/exe/FaceLandmarkImg/CMakeLists.txt index 9df2440..8ea140a 100644 --- a/exe/FaceLandmarkImg/CMakeLists.txt +++ b/exe/FaceLandmarkImg/CMakeLists.txt @@ -1,3 +1,6 @@ +#TBB library +include_directories(${TBB_ROOT_DIR}/include) + # Local libraries include_directories(${LandmarkDetector_SOURCE_DIR}/include) diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj b/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj index c92c3df..58a8b7b 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj @@ -123,7 +123,7 @@ Disabled WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) - AdvancedVectorExtensions2 + AdvancedVectorExtensions Console @@ -159,7 +159,7 @@ WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) false - AdvancedVectorExtensions2 + AdvancedVectorExtensions Console diff --git a/exe/FaceLandmarkVid/CMakeLists.txt b/exe/FaceLandmarkVid/CMakeLists.txt index 7274dde..6755fa1 100644 --- a/exe/FaceLandmarkVid/CMakeLists.txt +++ b/exe/FaceLandmarkVid/CMakeLists.txt @@ -1,3 +1,6 @@ +#TBB library +include_directories(${TBB_ROOT_DIR}/include) + add_executable(FaceLandmarkVid FaceLandmarkVid.cpp) # Local libraries diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj b/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj index 50848d0..facf547 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj @@ -125,7 +125,7 @@ WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) false - AdvancedVectorExtensions2 + AdvancedVectorExtensions Console @@ -165,7 +165,7 @@ $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) false Speed - AdvancedVectorExtensions2 + AdvancedVectorExtensions Console diff --git a/exe/FaceLandmarkVidMulti/CMakeLists.txt b/exe/FaceLandmarkVidMulti/CMakeLists.txt index 161b33b..7dabc9d 100644 --- a/exe/FaceLandmarkVidMulti/CMakeLists.txt +++ b/exe/FaceLandmarkVidMulti/CMakeLists.txt @@ -1,3 +1,6 @@ +#TBB library +include_directories(${TBB_ROOT_DIR}/include) + # Local libraries include_directories(${LandmarkDetector_SOURCE_DIR}/include) diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj index e2ab430..e187d58 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj @@ -113,7 +113,7 @@ Level3 Disabled $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) - AdvancedVectorExtensions2 + AdvancedVectorExtensions true @@ -142,7 +142,7 @@ true true $(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) - AdvancedVectorExtensions2 + AdvancedVectorExtensions MultiThreadedDLL diff --git a/exe/FeatureExtraction/CMakeLists.txt b/exe/FeatureExtraction/CMakeLists.txt index 5839c04..3c56149 100644 --- a/exe/FeatureExtraction/CMakeLists.txt +++ b/exe/FeatureExtraction/CMakeLists.txt @@ -1,3 +1,6 @@ +#TBB library +include_directories(${TBB_ROOT_DIR}/include) + add_executable(FeatureExtraction FeatureExtraction.cpp) # Local libraries diff --git a/exe/FeatureExtraction/FeatureExtraction.vcxproj b/exe/FeatureExtraction/FeatureExtraction.vcxproj index 6ee2834..8a3fc8c 100644 --- a/exe/FeatureExtraction/FeatureExtraction.vcxproj +++ b/exe/FeatureExtraction/FeatureExtraction.vcxproj @@ -124,7 +124,7 @@ WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) false - AdvancedVectorExtensions2 + AdvancedVectorExtensions Console @@ -163,7 +163,7 @@ $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) false Speed - AdvancedVectorExtensions2 + AdvancedVectorExtensions MultiThreadedDLL diff --git a/lib/3rdParty/dlib/dlib.vcxproj b/lib/3rdParty/dlib/dlib.vcxproj index 51445a4..94126b8 100644 --- a/lib/3rdParty/dlib/dlib.vcxproj +++ b/lib/3rdParty/dlib/dlib.vcxproj @@ -118,7 +118,7 @@ EnableFastChecks CompileAsCpp ProgramDatabase - AdvancedVectorExtensions2 + AdvancedVectorExtensions Sync Disabled Disabled @@ -175,7 +175,7 @@ $(SolutionDir)dlib/include/dlib/..;%(AdditionalIncludeDirectories) CompileAsCpp - AdvancedVectorExtensions2 + AdvancedVectorExtensions Sync AnySuitable Full diff --git a/lib/local/FaceAnalyser/CMakeLists.txt b/lib/local/FaceAnalyser/CMakeLists.txt index c08888f..f0081e7 100644 --- a/lib/local/FaceAnalyser/CMakeLists.txt +++ b/lib/local/FaceAnalyser/CMakeLists.txt @@ -1,3 +1,6 @@ +#TBB library +include_directories(${TBB_ROOT_DIR}/include) + include_directories(${BOOST_INCLUDE_DIR}) SET(SOURCE diff --git a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj index a7772f1..400502d 100644 --- a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj +++ b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj @@ -113,7 +113,7 @@ Disabled false ./include;$(SolutionDir)lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) - AdvancedVectorExtensions2 + AdvancedVectorExtensions true @@ -158,7 +158,7 @@ ./include;$(SolutionDir)lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) - AdvancedVectorExtensions2 + AdvancedVectorExtensions true diff --git a/lib/local/LandmarkDetector/CMakeLists.txt b/lib/local/LandmarkDetector/CMakeLists.txt index 16b8453..1c28678 100644 --- a/lib/local/LandmarkDetector/CMakeLists.txt +++ b/lib/local/LandmarkDetector/CMakeLists.txt @@ -1,3 +1,6 @@ +#TBB library +include_directories(${TBB_ROOT_DIR}/include) + include_directories(${BOOST_INCLUDE_DIR}) SET(SOURCE @@ -35,4 +38,4 @@ include_directories(${LandmarkDetector_SOURCE_DIR}/include) add_library( LandmarkDetector ${SOURCE} ${HEADERS}) install (TARGETS LandmarkDetector DESTINATION bin) -install (FILES HEADERS DESTINATION include) \ No newline at end of file +install (FILES HEADERS DESTINATION include) diff --git a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj index ea634cc..0a0c856 100644 --- a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj +++ b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj @@ -119,7 +119,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c NotUsing Level3 ProgramDatabase - AdvancedVectorExtensions2 + AdvancedVectorExtensions false /Zm300 %(AdditionalOptions) @@ -166,7 +166,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c Use Level3 ProgramDatabase - AdvancedVectorExtensions2 + AdvancedVectorExtensions Speed false /Zm300 %(AdditionalOptions) diff --git a/matlab_version/demo/face_image_demo_eyes.m b/matlab_version/demo/face_image_demo_eyes.m index 40a3714..05bf43f 100644 --- a/matlab_version/demo/face_image_demo_eyes.m +++ b/matlab_version/demo/face_image_demo_eyes.m @@ -9,22 +9,30 @@ addpath('../CCNF/'); [clmParams, pdm] = Load_CLM_params_wild(); -[clmParams_eye, pdm_eye] = Load_CLM_params_eye(); - % An accurate CCNF (or CLNF) model % [patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*_general.mat', [], [], clmParams); % A simpler (but less accurate SVR) [patches] = Load_Patch_Experts( '../models/general/', 'svr_patches_*_general.mat', [], [], clmParams); -[patches_eye] = Load_Patch_Experts( 'C:\Users\Tadas\Dropbox\AAM\patch_experts_eyes\svr_training\trained/', 'svr_patches_*_synth.mat', [], [], clmParams); +% Loading eye PDM and patch experts +[clmParams_eye, pdm_right_eye, pdm_left_eye] = Load_CLM_params_eye_28(); +[patches_right_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_right_eye.mat', [], [], clmParams_eye); +[patches_left_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_left_eye.mat', [], [], clmParams_eye); +clmParams_eye.multi_modal_types = patches_right_eye(1).multi_modal_types; +right_eye_inds = [43,44,45,46,47,48]; +left_eye_inds = [37,38,39,40,41,42]; + +right_eye_inds_synth = [9 11 13 15 17 19]; +left_eye_inds_synth = [9 11 13 15 17 19]; clmParams.multi_modal_types = patches(1).multi_modal_types; -clmParams_eye.multi_modal_types = patches_eye(1).multi_modal_types; - %% -root_dir = 'C:\Users\Tadas\Dropbox\AAM\test data\gaze_original\p00/'; -images = dir([root_dir, '*.jpg']); +% root_dir = 'C:\Users\Tadas\Dropbox\AAM\test data\gaze_original\p00/'; +% images = dir([root_dir, '*.jpg']); + +root_dir = './sample_eye_imgs/'; +images = dir([root_dir, '/*.png']); verbose = true; @@ -83,43 +91,36 @@ for img=1:numel(images) % shape correction for matlab format shape = shape + 1; + + % Perform eye fitting now + shape_r_eye = zeros(numel(pdm_right_eye.M)/3, 2); + shape_r_eye(right_eye_inds_synth,:) = shape(right_eye_inds, :); - if(verbose) + [ a, R, T, ~, l_params] = fit_PDM_ortho_proj_to_2D(pdm_right_eye.M, pdm_right_eye.E, pdm_right_eye.V, shape_r_eye); - % valid points to draw (not to draw self-occluded ones) - v_points = logical(patches(1).visibilities(view_used,:)); + bbox = [min(shape_r_eye(:,1)), min(shape_r_eye(:,2)), max(shape_r_eye(:,1)), max(shape_r_eye(:,2))]; - try - - plot(shape(v_points,1), shape(v_points',2),'.r','MarkerSize',20); - plot(shape(v_points,1), shape(v_points',2),'.b','MarkerSize',10); - - catch warn - - end - end - - % Map from detected landmarks to eye params - shape_r_eye = zeros(20,2); - shape_r_eye([9,11,13,15,17,19],:) = shape([43,44,45,46,47,48], :); - - [ a, R, T, ~, params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm_eye.M, pdm_eye.E, pdm_eye.V, shape_r_eye); - g_param = [a; Rot2Euler(R)'; T]; - l_param = params; - % Use the initial global and local params for clm fitting in the image - patches_eye(1).visibilities(1:8) = 0; - patches_eye(2).visibilities(1:8) = 0; - patches_eye(3).visibilities(1:8) = 0; - [shape_eye,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm_eye, patches_eye, clmParams_eye, 'gparam', g_param, 'lparam', l_param); + [shape_r_eye] = Fitting_from_bb(image, [], bbox, pdm_right_eye, patches_right_eye, clmParams_eye, 'gparam', g_param, 'lparam', l_params); - plot(shape_eye(:,1), shape_eye(:,2), '.g', 'MarkerSize',15); -% % Now do the eyes -% min_x = shape(43,1); -% max_x = shape(43,1); -% bbox_eye = shape(43,1) - + % Perform eye fitting now + shape_l_eye = zeros(numel(pdm_right_eye.M)/3, 2); + shape_l_eye(left_eye_inds_synth,:) = shape(left_eye_inds, :); + + [ a, R, T, ~, l_params] = fit_PDM_ortho_proj_to_2D(pdm_left_eye.M, pdm_left_eye.E, pdm_left_eye.V, shape_l_eye); + + bbox = [min(shape_l_eye(:,1)), min(shape_l_eye(:,2)), max(shape_l_eye(:,1)), max(shape_l_eye(:,2))]; + + g_param = [a; Rot2Euler(R)'; T]; + + [shape_l_eye] = Fitting_from_bb(image, [], bbox, pdm_left_eye, patches_left_eye, clmParams_eye, 'gparam', g_param, 'lparam', l_params); + + plot(shape_l_eye(9:20,1), shape_l_eye(9:20,2), '.g', 'MarkerSize',15); + plot(shape_l_eye(1:8,1), shape_l_eye(1:8,2), '.b', 'MarkerSize',15); + + plot(shape_r_eye(9:20,1), shape_r_eye(9:20,2), '.g', 'MarkerSize',15); + plot(shape_r_eye(1:8,1), shape_r_eye(1:8,2), '.b', 'MarkerSize',15); end hold off; diff --git a/matlab_version/demo/sample_eye_imgs/image_0037.png b/matlab_version/demo/sample_eye_imgs/image_0037.png new file mode 100644 index 0000000..34721de Binary files /dev/null and b/matlab_version/demo/sample_eye_imgs/image_0037.png differ diff --git a/matlab_version/demo/sample_eye_imgs/image_0038.png b/matlab_version/demo/sample_eye_imgs/image_0038.png new file mode 100644 index 0000000..af58c18 Binary files /dev/null and b/matlab_version/demo/sample_eye_imgs/image_0038.png differ diff --git a/matlab_version/demo/sample_eye_imgs/image_0039.png b/matlab_version/demo/sample_eye_imgs/image_0039.png new file mode 100644 index 0000000..a665a2d Binary files /dev/null and b/matlab_version/demo/sample_eye_imgs/image_0039.png differ diff --git a/matlab_version/demo/sample_eye_imgs/image_0093.png b/matlab_version/demo/sample_eye_imgs/image_0093.png new file mode 100644 index 0000000..defd5e9 Binary files /dev/null and b/matlab_version/demo/sample_eye_imgs/image_0093.png differ diff --git a/matlab_version/experiments_in_the_wild/Script_PDM_eyes.m b/matlab_version/experiments_in_the_wild/Script_PDM_eyes.m deleted file mode 100644 index e83a29d..0000000 --- a/matlab_version/experiments_in_the_wild/Script_PDM_eyes.m +++ /dev/null @@ -1,80 +0,0 @@ -function Script_PDM_eyes() - -addpath('../PDM_helpers/'); -addpath('../fitting/normxcorr2_mex_ALL'); -addpath('../fitting/'); -addpath('../CCNF/'); -addpath('../models/'); - -% Replace this with the location of in 300 faces in the wild data -if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) - root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; -else - root_test_data = 'F:/Dropbox/Dropbox/AAM/test data/'; -end - -[images, detections, labels] = Collect_wild_imgs(root_test_data); - -%% Fitting the model to the provided image - -% the default PDM to use -pdmLoc = ['../models/pdm/pdm_68_aligned_wild_eyes.mat']; - -load(pdmLoc); - -pdm = struct; -pdm.M = double(M); -pdm.E = double(E); -pdm.V = double(V); - -num_points = numel(M)/3; - -errors = zeros(numel(images),1); -shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); -labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); -errors_normed = zeros(numel(images),1); - -errors_left_eye = zeros(numel(images),1); -errors_right_eye = zeros(numel(images),1); - -tic -for i=1:numel(images) - - image = imread(images(i).img); - image_orig = image; - - if(size(image,3) == 3) - image = rgb2gray(image); - end - - labels_curr = squeeze(labels(i,:,:)); - - [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, labels_curr); - - shape = shapeOrtho; - shapes_all(:,:,i) = shapeOrtho; - labels_all(:,:,i) = labels_curr; - - if(mod(i, 200)==0) - fprintf('%d done\n', i ); - end - - valid_points = sum(squeeze(labels(i,:,:)),2) > 0; - valid_points(1:17) = 0; - - actualShape = squeeze(labels(i,:,:)); - errors(i) = sqrt(mean(sum((actualShape(valid_points,:) - shape(valid_points,:)).^2,2))); - width = ((max(actualShape(valid_points,1)) - min(actualShape(valid_points,1)))+(max(actualShape(valid_points,2)) - min(actualShape(valid_points,2))))/2; - errors_normed(i) = errors(i)/width; - - errors_left_eye(i) = compute_error_point_to_line_left_eye(actualShape, shapeOrtho, [0]); - errors_right_eye(i) = compute_error_point_to_line_right_eye(actualShape, shapeOrtho, [0]); - - if(errors_normed(i) > 0.035 || errors_left_eye(i) > 0.035 || errors_right_eye(i) > 0.035) - imshow(image);hold on; plot(shape(:,1), shape(:,2), '.g'); hold off; - end -end - -save('Errors_PDM_eyes.mat', 'errors_normed', 'errors_left_eye', 'errors_right_eye'); - -end diff --git a/matlab_version/experiments_in_the_wild/Script_PDM_general.m b/matlab_version/experiments_in_the_wild/Script_PDM_general.m deleted file mode 100644 index fb52e2e..0000000 --- a/matlab_version/experiments_in_the_wild/Script_PDM_general.m +++ /dev/null @@ -1,80 +0,0 @@ -function Script_PDM_general() - -addpath('../PDM_helpers/'); -addpath('../fitting/normxcorr2_mex_ALL'); -addpath('../fitting/'); -addpath('../CCNF/'); -addpath('../models/'); - -% Replace this with the location of in 300 faces in the wild data -if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) - root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; -else - root_test_data = 'F:/Dropbox/Dropbox/AAM/test data/'; -end - -[images, detections, labels] = Collect_wild_imgs(root_test_data); - -%% Fitting the model to the provided image - -% the default PDM to use -pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; - -load(pdmLoc); - -pdm = struct; -pdm.M = double(M); -pdm.E = double(E); -pdm.V = double(V); - -num_points = numel(M)/3; - -errors = zeros(numel(images),1); -shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); -labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); -errors_normed = zeros(numel(images),1); - -errors_left_eye = zeros(numel(images),1); -errors_right_eye = zeros(numel(images),1); - -tic -for i=1:numel(images) - - image = imread(images(i).img); - image_orig = image; - - if(size(image,3) == 3) - image = rgb2gray(image); - end - - labels_curr = squeeze(labels(i,:,:)); - - [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, labels_curr); - - shape = shapeOrtho; - shapes_all(:,:,i) = shapeOrtho; - labels_all(:,:,i) = labels_curr; - - if(mod(i, 200)==0) - fprintf('%d done\n', i ); - end - - valid_points = sum(squeeze(labels(i,:,:)),2) > 0; - valid_points(1:17) = 0; - - actualShape = squeeze(labels(i,:,:)); - errors(i) = sqrt(mean(sum((actualShape(valid_points,:) - shape(valid_points,:)).^2,2))); - width = ((max(actualShape(valid_points,1)) - min(actualShape(valid_points,1)))+(max(actualShape(valid_points,2)) - min(actualShape(valid_points,2))))/2; - errors_normed(i) = errors(i)/width; - - errors_left_eye(i) = compute_error_point_to_line_left_eye(actualShape, shapeOrtho, [0]); - errors_right_eye(i) = compute_error_point_to_line_right_eye(actualShape, shapeOrtho, [0]); - - if(errors_normed(i) > 0.035 || errors_left_eye(i) > 0.035 || errors_right_eye(i) > 0.035) - imshow(image);hold on; plot(shape(:,1), shape(:,2), '.g'); hold off; - end -end - -save('Errors_PDM_basic.mat', 'errors_normed', 'errors_left_eye', 'errors_right_eye'); - -end diff --git a/matlab_version/experiments_in_the_wild/compute_error_point_to_line_left_eye.m b/matlab_version/experiments_in_the_wild/compute_error_point_to_line_left_eye.m deleted file mode 100644 index 4b93b43..0000000 --- a/matlab_version/experiments_in_the_wild/compute_error_point_to_line_left_eye.m +++ /dev/null @@ -1,86 +0,0 @@ -function [ error_per_image ] = compute_error_point_to_line_left_eye( ground_truth_all, detected_points_all, occluded ) -%compute_error -% compute the average point-to-point Euclidean error normalized by the -% inter-ocular distance (measured as the Euclidean distance between the -% outer corners of the eyes) -% -% Inputs: -% grounth_truth_all, size: num_of_points x 2 x num_of_images -% detected_points_all, size: num_of_points x 2 x num_of_images -% Output: -% error_per_image, size: num_of_images x 1 - - - -right_eye_inds_from_68 = [37,38,39,40,41,42,37]; -right_eye_inds_from_28 = [9,11,13,15,17,19]; - -num_of_images = size(ground_truth_all,3); - -num_points_gt = size(ground_truth_all,1); - -num_points_det = size(detected_points_all,1); - -error_per_image = zeros(num_of_images,1); - -for i =1:num_of_images - - if(num_points_det == 6) - detected_points = detected_points_all(:,:,i); - elseif(num_points_det == 68 || num_points_det == 66) - detected_points = detected_points_all(right_eye_inds_from_68,:,i); - elseif(num_points_det == 28) - detected_points = detected_points_all(right_eye_inds_from_28,:,i); - elseif(num_points_det == 49) - - end - ground_truth_points = ground_truth_all(:,:,i); - - if(num_points_gt == 66 || num_points_gt == 68) - interocular_distance = norm(ground_truth_points(37,:)-ground_truth_points(46,:)); - ground_truth_points = ground_truth_points(right_eye_inds_from_68,:,:); - else - interocular_distance = norm(ground_truth_points(37-17,:)-ground_truth_points(46-17,:)); - ground_truth_points = ground_truth_points(right_eye_inds_from_68,:,:); - end - - sum=0; - for j=1:6 - - if(j== 1 || j == 6) - % eye corners should align perfectly - sum = sum + norm(detected_points(j,:)-ground_truth_points(j,:)); - else - % points between eye corners measured in distance to the two appropriate line - % segments - sum = sum + point_to_segments(detected_points(j,:), ground_truth_points(j-1:j+1,:)); - end - end - error_per_image(i) = sum/(6*interocular_distance); -end - -error_per_image = error_per_image(~occluded); - -end - -function seg_dist = point_to_segments(point, segments) - - seg_dists = zeros(size(segments, 1)-1,1); - - for i=1:size(segments, 1)-1 - - vec1 = point - segments(i,:); - vec2 = segments(i+1,:) - segments(i,:); - - d = (vec1 * vec2') / (norm(vec2)^2); - - if(d < 0) - seg_dists(i) = norm(vec1); - elseif(d > 1) - seg_dists(i) = norm(point - segments(i+1,:)); - else - seg_dists(i) = sqrt( norm(vec1)^2 - norm(d * vec2)^2); - end - end - seg_dist = min(seg_dists); -end \ No newline at end of file diff --git a/matlab_version/experiments_in_the_wild/compute_error_point_to_line_right_eye.m b/matlab_version/experiments_in_the_wild/compute_error_point_to_line_right_eye.m deleted file mode 100644 index b6dd94a..0000000 --- a/matlab_version/experiments_in_the_wild/compute_error_point_to_line_right_eye.m +++ /dev/null @@ -1,86 +0,0 @@ -function [ error_per_image ] = compute_error_point_to_line_right_eye( ground_truth_all, detected_points_all, occluded ) -%compute_error -% compute the average point-to-point Euclidean error normalized by the -% inter-ocular distance (measured as the Euclidean distance between the -% outer corners of the eyes) -% -% Inputs: -% grounth_truth_all, size: num_of_points x 2 x num_of_images -% detected_points_all, size: num_of_points x 2 x num_of_images -% Output: -% error_per_image, size: num_of_images x 1 - - - -right_eye_inds_from_68 = [43,44,45,46,47,48,43]; -right_eye_inds_from_28 = [9,11,13,15,17,19]; - -num_of_images = size(ground_truth_all,3); - -num_points_gt = size(ground_truth_all,1); - -num_points_det = size(detected_points_all,1); - -error_per_image = zeros(num_of_images,1); - -for i =1:num_of_images - - if(num_points_det == 6) - detected_points = detected_points_all(:,:,i); - elseif(num_points_det == 68 || num_points_det == 66) - detected_points = detected_points_all(right_eye_inds_from_68,:,i); - elseif(num_points_det == 28) - detected_points = detected_points_all(right_eye_inds_from_28,:,i); - elseif(num_points_det == 49) - - end - ground_truth_points = ground_truth_all(:,:,i); - - if(num_points_gt == 66 || num_points_gt == 68) - interocular_distance = norm(ground_truth_points(37,:)-ground_truth_points(46,:)); - ground_truth_points = ground_truth_points(right_eye_inds_from_68,:,:); - else - interocular_distance = norm(ground_truth_points(37-17,:)-ground_truth_points(46-17,:)); - ground_truth_points = ground_truth_points(right_eye_inds_from_68,:,:); - end - - sum=0; - for j=1:6 - - if(j== 1 || j == 6) - % eye corners should align perfectly - sum = sum + norm(detected_points(j,:)-ground_truth_points(j,:)); - else - % points between eye corners measured in distance to the two appropriate line - % segments - sum = sum + point_to_segments(detected_points(j,:), ground_truth_points(j-1:j+1,:)); - end - end - error_per_image(i) = sum/(6*interocular_distance); -end - -error_per_image = error_per_image(~occluded); - -end - -function seg_dist = point_to_segments(point, segments) - - seg_dists = zeros(size(segments, 1)-1,1); - - for i=1:size(segments, 1)-1 - - vec1 = point - segments(i,:); - vec2 = segments(i+1,:) - segments(i,:); - - d = (vec1 * vec2') / (norm(vec2)^2); - - if(d < 0) - seg_dists(i) = norm(vec1); - elseif(d > 1) - seg_dists(i) = norm(point - segments(i+1,:)); - else - seg_dists(i) = sqrt( norm(vec1)^2 - norm(d * vec2)^2); - end - end - seg_dist = min(seg_dists); -end \ No newline at end of file diff --git a/matlab_version/experiments_in_the_wild/Script_CLNF_general_eyes.m b/matlab_version/experiments_in_the_wild/hierarch_checks/Script_CLNF_general_eyes.m similarity index 100% rename from matlab_version/experiments_in_the_wild/Script_CLNF_general_eyes.m rename to matlab_version/experiments_in_the_wild/hierarch_checks/Script_CLNF_general_eyes.m diff --git a/matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_eyes.m b/matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_eyes.m index 955524d..e83a29d 100644 --- a/matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_eyes.m +++ b/matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_eyes.m @@ -27,8 +27,12 @@ pdm.M = double(M); pdm.E = double(E); pdm.V = double(V); +num_points = numel(M)/3; + +errors = zeros(numel(images),1); shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +errors_normed = zeros(numel(images),1); errors_left_eye = zeros(numel(images),1); errors_right_eye = zeros(numel(images),1); @@ -36,30 +40,41 @@ errors_right_eye = zeros(numel(images),1); tic for i=1:numel(images) + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + labels_curr = squeeze(labels(i,:,:)); - [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D_no_reg(pdm.M, pdm.E, pdm.V, labels_curr); + [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, labels_curr); + shape = shapeOrtho; shapes_all(:,:,i) = shapeOrtho; labels_all(:,:,i) = labels_curr; - if(mod(i, 100)==0) + if(mod(i, 200)==0) fprintf('%d done\n', i ); end + valid_points = sum(squeeze(labels(i,:,:)),2) > 0; + valid_points(1:17) = 0; + actualShape = squeeze(labels(i,:,:)); + errors(i) = sqrt(mean(sum((actualShape(valid_points,:) - shape(valid_points,:)).^2,2))); + width = ((max(actualShape(valid_points,1)) - min(actualShape(valid_points,1)))+(max(actualShape(valid_points,2)) - min(actualShape(valid_points,2))))/2; + errors_normed(i) = errors(i)/width; errors_left_eye(i) = compute_error_point_to_line_left_eye(actualShape, shapeOrtho, [0]); errors_right_eye(i) = compute_error_point_to_line_right_eye(actualShape, shapeOrtho, [0]); - if(errors_left_eye(i) > 0.02 || errors_right_eye(i) > 0.02) - plot(shapeOrtho(:,1), -shapeOrtho(:,2), 'r.'); hold on; - axis equal; - plot(labels_curr(:,1), -labels_curr(:,2), 'g.'); hold off; + if(errors_normed(i) > 0.035 || errors_left_eye(i) > 0.035 || errors_right_eye(i) > 0.035) + imshow(image);hold on; plot(shape(:,1), shape(:,2), '.g'); hold off; end - end -save('Errors_PDM_eyes.mat', 'errors_left_eye', 'errors_right_eye'); +save('Errors_PDM_eyes.mat', 'errors_normed', 'errors_left_eye', 'errors_right_eye'); end diff --git a/matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_general.m b/matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_general.m index 119ffd2..fb52e2e 100644 --- a/matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_general.m +++ b/matlab_version/experiments_in_the_wild/hierarch_checks/Script_PDM_general.m @@ -1,4 +1,4 @@ -function Script_PDM_eyes() +function Script_PDM_general() addpath('../PDM_helpers/'); addpath('../fitting/normxcorr2_mex_ALL'); @@ -27,8 +27,12 @@ pdm.M = double(M); pdm.E = double(E); pdm.V = double(V); +num_points = numel(M)/3; + +errors = zeros(numel(images),1); shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +errors_normed = zeros(numel(images),1); errors_left_eye = zeros(numel(images),1); errors_right_eye = zeros(numel(images),1); @@ -36,31 +40,41 @@ errors_right_eye = zeros(numel(images),1); tic for i=1:numel(images) + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + labels_curr = squeeze(labels(i,:,:)); - [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D_no_reg(pdm.M, pdm.E, pdm.V, labels_curr); + [ a, R, T, ~, l_params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, labels_curr); + shape = shapeOrtho; shapes_all(:,:,i) = shapeOrtho; labels_all(:,:,i) = labels_curr; - if(mod(i, 100)==0) + if(mod(i, 200)==0) fprintf('%d done\n', i ); end + valid_points = sum(squeeze(labels(i,:,:)),2) > 0; + valid_points(1:17) = 0; + actualShape = squeeze(labels(i,:,:)); + errors(i) = sqrt(mean(sum((actualShape(valid_points,:) - shape(valid_points,:)).^2,2))); + width = ((max(actualShape(valid_points,1)) - min(actualShape(valid_points,1)))+(max(actualShape(valid_points,2)) - min(actualShape(valid_points,2))))/2; + errors_normed(i) = errors(i)/width; errors_left_eye(i) = compute_error_point_to_line_left_eye(actualShape, shapeOrtho, [0]); errors_right_eye(i) = compute_error_point_to_line_right_eye(actualShape, shapeOrtho, [0]); - - if(errors_left_eye(i) > 0.02 || errors_right_eye(i) > 0.02) - plot(shapeOrtho(:,1), -shapeOrtho(:,2), 'r.'); hold on; - axis equal; - plot(labels_curr(:,1), -labels_curr(:,2), 'g.'); hold off; + if(errors_normed(i) > 0.035 || errors_left_eye(i) > 0.035 || errors_right_eye(i) > 0.035) + imshow(image);hold on; plot(shape(:,1), shape(:,2), '.g'); hold off; end - end -save('Errors_PDM_basic.mat', 'errors_left_eye', 'errors_right_eye'); +save('Errors_PDM_basic.mat', 'errors_normed', 'errors_left_eye', 'errors_right_eye'); end diff --git a/matlab_version/experiments_in_the_wild/compute_brow_error_to_line.m b/matlab_version/experiments_in_the_wild/hierarch_checks/compute_brow_error_to_line.m similarity index 100% rename from matlab_version/experiments_in_the_wild/compute_brow_error_to_line.m rename to matlab_version/experiments_in_the_wild/hierarch_checks/compute_brow_error_to_line.m diff --git a/matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_left_eye.m b/matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_left_eye.m index c2edd30..4b93b43 100644 --- a/matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_left_eye.m +++ b/matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_left_eye.m @@ -59,9 +59,7 @@ for i =1:num_of_images error_per_image(i) = sum/(6*interocular_distance); end -if(nargin > 2) - error_per_image = error_per_image(~occluded); -end +error_per_image = error_per_image(~occluded); end diff --git a/matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_right_eye.m b/matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_right_eye.m index e03fb47..b6dd94a 100644 --- a/matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_right_eye.m +++ b/matlab_version/experiments_in_the_wild/hierarch_checks/compute_error_point_to_line_right_eye.m @@ -59,9 +59,7 @@ for i =1:num_of_images error_per_image(i) = sum/(6*interocular_distance); end -if(nargin > 2) - error_per_image = error_per_image(~occluded); -end +error_per_image = error_per_image(~occluded); end diff --git a/matlab_version/models/Load_CLM_params_eye_28.m b/matlab_version/models/Load_CLM_params_eye_28.m new file mode 100644 index 0000000..a5b8ea5 --- /dev/null +++ b/matlab_version/models/Load_CLM_params_eye_28.m @@ -0,0 +1,38 @@ +function [ clmParams, pdm_right, pdm_left ] = Load_CLM_params_eye_28() +%LOAD_CLM_PARAMS_WILD Summary of this function goes here +% Detailed explanation goes here + clmParams.window_size = [17,17; 15,15; 13,13;]; + clmParams.numPatchIters = size(clmParams.window_size,1); + + % the PDM created from in the wild data + pdmLoc = ['../models/hierarch_pdm/pdm_28_r_eye.mat']; + + load(pdmLoc); + + pdm_right = struct; + pdm_right.M = double(M); + pdm_right.E = double(E); + pdm_right.V = double(V); + + pdmLoc = ['../models/hierarch_pdm/pdm_28_l_eye.mat']; + + load(pdmLoc); + + pdm_left = struct; + pdm_left.M = double(M); + pdm_left.E = double(E); + pdm_left.V = double(V); + + % the default model parameters to use + clmParams.regFactor = 2.0; + clmParams.sigmaMeanShift = 1.5; + clmParams.tikhonov_factor = 0; + + clmParams.startScale = 1; + clmParams.num_RLMS_iter = 10; + clmParams.fTol = 0.01; + clmParams.useMultiScale = true; + clmParams.use_multi_modal = 0; + clmParams.tikhonov_factor = 0; +end + diff --git a/matlab_version/models/hierarch/ccnf_patches_0.75_synth_left_eye.mat b/matlab_version/models/hierarch/ccnf_patches_0.75_synth_left_eye.mat new file mode 100644 index 0000000..919a6a3 Binary files /dev/null and b/matlab_version/models/hierarch/ccnf_patches_0.75_synth_left_eye.mat differ diff --git a/matlab_version/models/hierarch/ccnf_patches_0.75_synth_right_eye.mat b/matlab_version/models/hierarch/ccnf_patches_0.75_synth_right_eye.mat new file mode 100644 index 0000000..60244e2 Binary files /dev/null and b/matlab_version/models/hierarch/ccnf_patches_0.75_synth_right_eye.mat differ diff --git a/matlab_version/models/hierarch/ccnf_patches_1.00_synth_left_eye.mat b/matlab_version/models/hierarch/ccnf_patches_1.00_synth_left_eye.mat new file mode 100644 index 0000000..9969757 Binary files /dev/null and b/matlab_version/models/hierarch/ccnf_patches_1.00_synth_left_eye.mat differ diff --git a/matlab_version/models/hierarch/ccnf_patches_1.00_synth_right_eye.mat b/matlab_version/models/hierarch/ccnf_patches_1.00_synth_right_eye.mat new file mode 100644 index 0000000..57c42df Binary files /dev/null and b/matlab_version/models/hierarch/ccnf_patches_1.00_synth_right_eye.mat differ diff --git a/matlab_version/models/hierarch/ccnf_patches_1.50_synth_left_eye.mat b/matlab_version/models/hierarch/ccnf_patches_1.50_synth_left_eye.mat new file mode 100644 index 0000000..21170af Binary files /dev/null and b/matlab_version/models/hierarch/ccnf_patches_1.50_synth_left_eye.mat differ diff --git a/matlab_version/models/hierarch/ccnf_patches_1.50_synth_right_eye.mat b/matlab_version/models/hierarch/ccnf_patches_1.50_synth_right_eye.mat new file mode 100644 index 0000000..deec91b Binary files /dev/null and b/matlab_version/models/hierarch/ccnf_patches_1.50_synth_right_eye.mat differ diff --git a/matlab_version/models/hierarch_pdm/pdm_28_l_eye.mat b/matlab_version/models/hierarch_pdm/pdm_28_l_eye.mat new file mode 100644 index 0000000..3c224d6 Binary files /dev/null and b/matlab_version/models/hierarch_pdm/pdm_28_l_eye.mat differ diff --git a/matlab_version/models/hierarch_pdm/pdm_28_r_eye.mat b/matlab_version/models/hierarch_pdm/pdm_28_r_eye.mat new file mode 100644 index 0000000..ea364f0 Binary files /dev/null and b/matlab_version/models/hierarch_pdm/pdm_28_r_eye.mat differ