369 lines
13 KiB
Matlab
369 lines
13 KiB
Matlab
function Create_data_train()
|
|
|
|
load '../models/pdm/pdm_68_aligned_menpo';
|
|
load '../models/tri_68.mat';
|
|
|
|
% This script uses the same format used for patch expert training, and
|
|
% expects the data to be there (this can be found in
|
|
% https://github.com/TadasBaltrusaitis/CCNF)
|
|
|
|
% Replace with your location of training data
|
|
dataset_loc = 'C:\Users\tbaltrus\Documents\CCNF\patch_experts\data_preparation/prepared_data/';
|
|
addpath('../PDM_helpers/');
|
|
addpath('./paw_helpers/');
|
|
|
|
% Collect Menpo, Multi-PIE and 300W data for training the validator
|
|
scale = '0.5';
|
|
prefix_menpo= 'menpo_train_';
|
|
prefix_mpie_300W = 'combined_';
|
|
|
|
% Find the available positive training data
|
|
data_files = dir(sprintf('%s/%s%s*.mat', dataset_loc, prefix_menpo, scale));
|
|
data_files_c = dir(sprintf('%s/%s%s*.mat', dataset_loc, prefix_mpie_300W, scale));
|
|
|
|
centres_all = [];
|
|
for i=1:numel(data_files)
|
|
|
|
% Load the orientation of the training data
|
|
load([dataset_loc, '/', data_files(i).name], 'centres');
|
|
centres_all = cat(1, centres_all, centres);
|
|
|
|
end
|
|
|
|
% Construct mirror indices (which views need to be flipped to create other
|
|
% profile training data)
|
|
mirror_inds = zeros(size(centres_all,1), 1);
|
|
for i=1:numel(data_files)
|
|
|
|
% mirrored image has inverse yaw
|
|
mirrored_centre = centres_all(i,:);
|
|
mirrored_centre(2) = -mirrored_centre(2);
|
|
|
|
% if mirrored version has same orientation, do not need mirroring
|
|
if(~isequal(mirrored_centre, centres_all(i,:)))
|
|
|
|
centres_all = cat(1, centres_all, mirrored_centre);
|
|
mirror_inds = cat(1, mirror_inds, i);
|
|
|
|
end
|
|
|
|
end
|
|
|
|
% Replace with your location of training data
|
|
outputLocation = 'D:\Datasets/detection_validation/prep_data/';
|
|
|
|
num_more_neg = 10;
|
|
|
|
% Make sure same data generated all the time
|
|
rng(0);
|
|
|
|
% Negative samples from teh INRIAPerson dataset
|
|
neg_image_loc = 'D:\Datasets\INRIAPerson\INRIAPerson\Train\neg/';
|
|
|
|
neg_images = cat(1,dir([neg_image_loc, '/*.jpg']),dir([neg_image_loc, '/*.png']));
|
|
|
|
max_img_used = 8000;
|
|
|
|
% do it separately for centers due to memory limitations
|
|
for r=1:size(centres_all,1)
|
|
|
|
a_mod = 0.4;
|
|
|
|
mirror = false;
|
|
|
|
if(mirror_inds(r) ~= 0 )
|
|
mirror = true;
|
|
label_mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
|
|
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
|
|
61,65;62,64;68,66];
|
|
load([dataset_loc, '/', data_files_c(mirror_inds(r)).name]);
|
|
all_images_t = all_images;
|
|
landmark_locations_t = landmark_locations;
|
|
visiIndex_t = visiIndex;
|
|
|
|
load([dataset_loc, '/', data_files(mirror_inds(r)).name]);
|
|
|
|
% Combining Menpo + MPIE + 300W
|
|
all_images = cat(1, all_images, all_images_t);
|
|
landmark_locations = cat(1, landmark_locations, landmark_locations_t);
|
|
% Taking a subset of visibilities from all the datasets
|
|
visiIndex = visiIndex_t & visiIndex;
|
|
else
|
|
load([dataset_loc, '/', data_files_c(r).name]);
|
|
all_images_t = all_images;
|
|
landmark_locations_t = landmark_locations;
|
|
visiIndex_t = visiIndex;
|
|
|
|
load([dataset_loc, '/', data_files(r).name]);
|
|
all_images = cat(1, all_images, all_images_t);
|
|
landmark_locations = cat(1, landmark_locations, landmark_locations_t);
|
|
visiIndex = visiIndex_t & visiIndex;
|
|
end
|
|
|
|
visiCurrent = logical(visiIndex);
|
|
|
|
if(mirror)
|
|
centres = [centres(1), -centres(2), -centres(3)];
|
|
tmp1 = visiCurrent(label_mirror_inds(:,1));
|
|
tmp2 = visiCurrent(label_mirror_inds(:,2));
|
|
visiCurrent(label_mirror_inds(:,2)) = tmp1;
|
|
visiCurrent(label_mirror_inds(:,1)) = tmp2;
|
|
end
|
|
|
|
visibleVerts = 1:numel(visiCurrent);
|
|
visibleVerts = visibleVerts(visiCurrent)-1;
|
|
|
|
% Correct the triangulation to take into account the vertex
|
|
% visibilities
|
|
triangulation = [];
|
|
|
|
shape = a_mod * Euler2Rot(centres * pi/180) * reshape(M, numel(M)/3, 3)';
|
|
shape = shape';
|
|
|
|
for i=1:size(T,1)
|
|
visib = 0;
|
|
for j=1:numel(visibleVerts)
|
|
if(T(i,1)==visibleVerts(j))
|
|
visib = visib+1;
|
|
end
|
|
if(T(i,2)==visibleVerts(j))
|
|
visib = visib+1;
|
|
end
|
|
if(T(i,3)==visibleVerts(j))
|
|
visib = visib+1;
|
|
end
|
|
end
|
|
|
|
% Only if all three of the vertices are visible
|
|
if(visib == 3)
|
|
|
|
% Also want to remove triangles facing the wrong way (self occluded)
|
|
v1 = [shape(T(i,1)+1,1), shape(T(i,1)+1,2), shape(T(i,1)+1,3)];
|
|
v2 = [shape(T(i,2)+1,1), shape(T(i,2)+1,2), shape(T(i,2)+1,3)];
|
|
v3 = [shape(T(i,3)+1,1), shape(T(i,3)+1,2), shape(T(i,3)+1,3)];
|
|
normal = cross((v2-v1), v3 - v2);
|
|
normal = normal / norm(normal);
|
|
direction = normal * [0,0,1]';
|
|
|
|
% And only if the triangle is facing the camera
|
|
if(direction > 0)
|
|
triangulation = cat(1, triangulation, T(i,:));
|
|
end
|
|
end
|
|
end
|
|
|
|
% Initialise the warp
|
|
[ alphas, betas, triX, mask, minX, minY, nPix ] = InitialisePieceWiseAffine(triangulation, shape);
|
|
|
|
mask = logical(mask);
|
|
|
|
imgs_to_use = randperm(size(landmark_locations, 1));
|
|
|
|
if(size(landmark_locations, 1) > max_img_used)
|
|
imgs_to_use = imgs_to_use(1:max_img_used);
|
|
end
|
|
|
|
% Extracting relevant filenames
|
|
examples = zeros(numel(imgs_to_use) * (num_more_neg+1), nPix);
|
|
errors = zeros(numel(imgs_to_use) * (num_more_neg+1), 1);
|
|
|
|
unused_pos = 0;
|
|
|
|
curr_filled = 0;
|
|
|
|
for j=imgs_to_use
|
|
|
|
labels = squeeze(landmark_locations(j,:,:));
|
|
|
|
img = squeeze(all_images(j,:,:));
|
|
|
|
if(mirror)
|
|
img = fliplr(img);
|
|
imgSize = size(img);
|
|
flippedLbls = labels;
|
|
flippedLbls(:,1) = imgSize(1) - flippedLbls(:,1) + 1;
|
|
tmp1 = flippedLbls(label_mirror_inds(:,1),:);
|
|
tmp2 = flippedLbls(label_mirror_inds(:,2),:);
|
|
flippedLbls(label_mirror_inds(:,2),:) = tmp1;
|
|
flippedLbls(label_mirror_inds(:,1),:) = tmp2;
|
|
labels = flippedLbls;
|
|
end
|
|
|
|
% If for some reason some of the labels are not visible in the
|
|
% current sample skip this label
|
|
non_existent_labels = labels(:,1)==0 | labels(:,2)==0;
|
|
non_existent_inds = find(non_existent_labels)-1;
|
|
if(numel(intersect(triangulation(:), non_existent_inds)) > 0)
|
|
unused_pos = unused_pos + 1;
|
|
continue;
|
|
end
|
|
|
|
% Centering the pixel so that 0,0 is center of the top left pixel
|
|
labels = labels - 1;
|
|
|
|
curr_filled = curr_filled + 1;
|
|
[features] = ExtractFaceFeatures(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
|
|
% sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
|
|
examples(curr_filled,:) = features;
|
|
errors(curr_filled,:) = 0;
|
|
|
|
% Extract the correct PDM parameters for the model (we will perturb
|
|
% them for some negative examples)
|
|
[ a_orig, R_orig, trans_orig, ~, params_orig] = fit_PDM_ortho_proj_to_2D(M, E, V, labels);
|
|
eul_orig = Rot2Euler(R_orig);
|
|
|
|
% a slightly perturbed example, too tight
|
|
% from 0.3 to 0.9
|
|
a_mod = a_orig * (0.6 + (randi(7) - 4)*0.1);
|
|
p_global = [a_mod; eul_orig'; trans_orig];
|
|
|
|
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
|
|
labels_mod = labels_mod(:,1:2);
|
|
|
|
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
|
|
% sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
|
|
|
|
curr_filled = curr_filled + 1;
|
|
examples(curr_filled,:) = features;
|
|
|
|
% Compute the badness of fit
|
|
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
|
|
errors(curr_filled,:) = error;
|
|
|
|
% a slightly perturbed example, too broad
|
|
% from 1.2 to 0.6
|
|
a_mod = a_orig * (1.4 + (randi(5) - 3)*0.1);
|
|
p_global = [a_mod; eul_orig'; trans_orig];
|
|
|
|
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
|
|
labels_mod = labels_mod(:,1:2);
|
|
|
|
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
|
|
% sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
|
|
|
|
curr_filled = curr_filled + 1;
|
|
examples(curr_filled,:) = features;
|
|
|
|
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
|
|
errors(curr_filled,:) = error;
|
|
|
|
% A somewhat offset example
|
|
|
|
trans_mod = trans_orig + randn(2,1) * 20;
|
|
p_global = [a_orig; eul_orig'; trans_mod];
|
|
|
|
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
|
|
labels_mod = labels_mod(:,1:2);
|
|
|
|
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
|
|
|
|
curr_filled = curr_filled + 1;
|
|
examples(curr_filled,:) = features;
|
|
|
|
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
|
|
errors(curr_filled,:) = error;
|
|
|
|
% A rotated sample
|
|
eul_mod = eul_orig + randn(1,3)*0.3;
|
|
p_global = [a_orig; eul_mod'; trans_orig];
|
|
|
|
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
|
|
labels_mod = labels_mod(:,1:2);
|
|
|
|
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
|
|
|
|
curr_filled = curr_filled + 1;
|
|
examples(curr_filled,:) = features;
|
|
|
|
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
|
|
errors(curr_filled,:) = error;
|
|
|
|
% A sample with modified shape parameters
|
|
p_global = [a_orig; eul_orig'; trans_orig];
|
|
params_mod = params_orig + randn(size(params_orig)).*sqrt(E);
|
|
labels_mod = GetShapeOrtho(M, V, params_mod, p_global);
|
|
labels_mod = labels_mod(:,1:2);
|
|
|
|
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
|
|
|
|
curr_filled = curr_filled + 1;
|
|
examples(curr_filled,:) = features;
|
|
|
|
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
|
|
errors(curr_filled,:) = error;
|
|
|
|
% pick a random image from negative inriaperson dataset, use original location if
|
|
% first, otherwhise resize it to fit
|
|
for n=6:num_more_neg
|
|
n_img = randi(numel(neg_images));
|
|
|
|
neg_image = imread([neg_image_loc, neg_images(n_img).name]);
|
|
|
|
if(size(neg_image,3) == 3)
|
|
neg_image = rgb2gray(neg_image);
|
|
end
|
|
|
|
[h_neg, w_neg] = size(neg_image);
|
|
|
|
% if the current labels fit just use them, if not, then resize
|
|
% to fit
|
|
max_x = max(labels(:,1));
|
|
max_y = max(labels(:,2));
|
|
|
|
if(max_x > w_neg || max_y > h_neg)
|
|
neg_image = imresize(neg_image, [max_y, max_x]);
|
|
end
|
|
|
|
[features] = ExtractFaceFeatures(neg_image, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
|
|
|
|
curr_filled = curr_filled + 1;
|
|
examples(curr_filled,:) = features;
|
|
|
|
% Set high error to 3
|
|
errors(curr_filled,:) = 3;
|
|
end
|
|
|
|
|
|
if(mod(curr_filled, 10) == 0)
|
|
fprintf('%d/%d done\n', curr_filled/(num_more_neg+1), numel(imgs_to_use));
|
|
end
|
|
% add the pos example to the background
|
|
|
|
end
|
|
|
|
examples = examples(1:curr_filled,:);
|
|
errors = errors(1:curr_filled);
|
|
|
|
filename = sprintf('%s/face_validator_train_%d.mat', outputLocation, r);
|
|
save(filename, 'examples', 'errors', 'alphas', 'betas', 'triangulation', 'minX', 'minY', 'nPix', 'shape', 'triX', 'mask', 'centres');
|
|
|
|
|
|
end
|
|
|
|
end
|
|
|
|
function [features] = ExtractFaceFeatures(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY)
|
|
|
|
% Make sure labels are within range
|
|
[hRes, wRes] = size(img);
|
|
labels(labels(:,1) < 0,1) = 0;
|
|
labels(labels(:,2) < 0,2) = 0;
|
|
|
|
labels(labels(:,1) > wRes-1,1) = wRes-1;
|
|
labels(labels(:,2) > hRes-1,2) = hRes-1;
|
|
|
|
crop_img = Crop(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
|
|
crop_img(isnan(crop_img)) = 0;
|
|
|
|
% vectorised version
|
|
features = reshape(crop_img(logical(mask)), 1, nPix);
|
|
|
|
% normalisations
|
|
features = (features - mean(features));
|
|
norms = std(features);
|
|
if(norms==0)
|
|
norms = 1;
|
|
end
|
|
features = features / norms;
|
|
|
|
end |