- some cleanup with depth

This commit is contained in:
Tadas Baltrusaitis 2017-10-21 12:15:22 +01:00
parent 1cfb765bb9
commit e71d13a22a
16 changed files with 15 additions and 122 deletions

View file

@ -238,7 +238,6 @@ int main (int argc, char **argv)
{ {
// Reading the images // Reading the images
cv::Mat_<float> depth_image;
cv::Mat_<uchar> grayscale_image; cv::Mat_<uchar> grayscale_image;
cv::Mat disp_image = captured_image.clone(); cv::Mat disp_image = captured_image.clone();
@ -398,12 +397,6 @@ int main (int argc, char **argv)
{ {
cv::namedWindow("tracking_result",1); cv::namedWindow("tracking_result",1);
cv::imshow("tracking_result", disp_image); cv::imshow("tracking_result", disp_image);
if(!depth_image.empty())
{
// Division needed for visualisation purposes
imshow("depth", depth_image/2000.0);
}
} }
// output the tracked video // output the tracked video

View file

@ -232,7 +232,7 @@ int main (int argc, char **argv)
// Get the input output file parameters // Get the input output file parameters
// Indicates that rotation should be with respect to camera or world coordinates // Indicates that rotation should be with respect to camera or world coordinates
bool use_camera_coordinates; bool use_camera_coordinates = false;
string output_codec; //not used but should string output_codec; //not used but should
LandmarkDetector::get_video_input_output_params(input_files, output_files, tracked_videos_output, use_camera_coordinates, output_codec, arguments); LandmarkDetector::get_video_input_output_params(input_files, output_files, tracked_videos_output, use_camera_coordinates, output_codec, arguments);

View file

@ -51,7 +51,7 @@ class CCNF_neuron{
public: public:
// Type of patch (0=raw,1=grad,3=depth, other types besides raw are not actually used now) // Type of patch (0=raw,1=grad other types besides raw are not actually used now)
int neuron_type; int neuron_type;
// scaling of weights (needed as the energy of neuron might not be 1) // scaling of weights (needed as the energy of neuron might not be 1)
@ -112,7 +112,7 @@ public:
void Read(std::ifstream &stream, std::vector<int> window_sizes, std::vector<std::vector<cv::Mat_<float> > > sigma_components); void Read(std::ifstream &stream, std::vector<int> window_sizes, std::vector<std::vector<cv::Mat_<float> > > sigma_components);
// actual work (can pass in an image and a potential depth image, if the CCNF is trained with depth) // actual work (can pass in an image)
void Response(cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response); void Response(cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
// Helper function to compute relevant sigmas // Helper function to compute relevant sigmas

View file

@ -53,7 +53,7 @@ namespace LandmarkDetector
// Helper functions for parsing the inputs // Helper functions for parsing the inputs
//============================================================================================= //=============================================================================================
void get_video_input_output_params(vector<string> &input_video_file, vector<string> &output_files, void get_video_input_output_params(vector<string> &input_video_file, vector<string> &output_files,
vector<string> &output_video_files, bool& world_coordinates_pose, string &output_codec, vector<string> &arguments); vector<string> &output_video_files, bool& camera_coordinates_pose, string &output_codec, vector<string> &arguments);
void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector<string> &arguments); void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector<string> &arguments);

View file

@ -78,9 +78,8 @@ class SVR_patch_expert{
// Reading in the patch expert // Reading in the patch expert
void Read(std::ifstream &stream); void Read(std::ifstream &stream);
// The actual response computation from intensity or depth (for CLM-Z) // The actual response computation from intensity
void Response(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response); void Response(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
void ResponseDepth(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
}; };
//=========================================================================== //===========================================================================
@ -105,9 +104,8 @@ class Multi_SVR_patch_expert{
void Read(std::ifstream &stream); void Read(std::ifstream &stream);
// actual response computation from intensity of depth (for CLM-Z) // actual response computation from intensity
void Response(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response); void Response(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
void ResponseDepth(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
}; };
} }

View file

@ -195,72 +195,6 @@ void SVR_patch_expert::Response(const cv::Mat_<float>& area_of_interest, cv::Mat
} }
void SVR_patch_expert::ResponseDepth(const cv::Mat_<float>& area_of_interest, cv::Mat_<float> &response)
{
// How big the response map will be
int response_height = area_of_interest.rows - weights.rows + 1;
int response_width = area_of_interest.cols - weights.cols + 1;
// the patch area on which we will calculate reponses
cv::Mat_<float> normalised_area_of_interest;
if(response.rows != response_height || response.cols != response_width)
{
response.create(response_height, response_width);
}
if(type == 0)
{
// Perform normalisation across whole patch
cv::Scalar mean;
cv::Scalar std;
// ignore missing values
cv::Mat_<uchar> mask = area_of_interest > 0;
cv::meanStdDev(area_of_interest, mean, std, mask);
// if all values the same don't divide by 0
if(std[0] == 0)
{
std[0] = 1;
}
normalised_area_of_interest = (area_of_interest - mean[0]) / std[0];
// Set the invalid pixels to 0
normalised_area_of_interest.setTo(0, mask == 0);
}
else
{
printf("ERROR(%s,%d): Unsupported patch type %d!\n", __FILE__,__LINE__,type);
abort();
}
cv::Mat_<float> svr_response;
// The empty matrix as we don't pass precomputed dft's of image
cv::Mat_<double> empty_matrix_0(0,0,0.0);
cv::Mat_<float> empty_matrix_1(0,0,0.0);
cv::Mat_<float> empty_matrix_2(0,0,0.0);
// Efficient calc of patch expert response across the area of interest
matchTemplate_m(normalised_area_of_interest, empty_matrix_0, empty_matrix_1, empty_matrix_2, weights, weights_dfts, svr_response, CV_TM_CCOEFF);
response.create(svr_response.size());
cv::MatIterator_<float> p = response.begin();
cv::MatIterator_<float> q1 = svr_response.begin(); // respone for each pixel
cv::MatIterator_<float> q2 = svr_response.end();
while(q1 != q2)
{
// the SVR response passed through a logistic regressor
*p++ = 1.0/(1.0 + exp( -(*q1++ * scaling + bias )));
}
}
// Copy constructor // Copy constructor
Multi_SVR_patch_expert::Multi_SVR_patch_expert(const Multi_SVR_patch_expert& other) : svr_patch_experts(other.svr_patch_experts) Multi_SVR_patch_expert::Multi_SVR_patch_expert(const Multi_SVR_patch_expert& other) : svr_patch_experts(other.svr_patch_experts)
{ {
@ -321,17 +255,3 @@ void Multi_SVR_patch_expert::Response(const cv::Mat_<float> &area_of_interest, c
} }
void Multi_SVR_patch_expert::ResponseDepth(const cv::Mat_<float>& area_of_interest, cv::Mat_<float>& response)
{
int response_height = area_of_interest.rows - height + 1;
int response_width = area_of_interest.cols - width + 1;
if(response.rows != response_height || response.cols != response_width)
{
response.create(response_height, response_width);
}
// With depth patch experts only do raw data modality
svr_patch_experts[0].ResponseDepth(area_of_interest, response);
}
//===========================================================================

View file

@ -1,2 +1,2 @@
Mean error, median error Mean error, median error
9.469, 8.773 9.405, 8.580

View file

@ -1,4 +1,4 @@
Dataset and model, pitch, yaw, roll, mean, median Dataset and model, pitch, yaw, roll, mean, median
biwi error: 7.092, 5.170, 4.657, 5.640, 2.607 biwi error: 7.092, 5.170, 4.657, 5.640, 2.607
bu error: 2.769, 4.105, 2.569, 3.147, 2.118 bu error: 2.762, 3.328, 2.249, 2.780, 1.963
ict error: 3.489, 3.632, 3.538, 3.553, 2.029 ict error: 3.529, 3.873, 2.688, 3.363, 1.931

View file

@ -1,4 +1,4 @@
function [output_dir] = run_biwi_experiment(rootDir, biwiDir, verbose, depth, varargin) function [output_dir] = run_biwi_experiment(rootDir, biwiDir, verbose, varargin)
% Biwi dataset experiment % Biwi dataset experiment
if(isunix) if(isunix)
@ -11,10 +11,6 @@ output_dir = 'experiments/biwi_out';
dbSeqDir = dir([rootDir biwiDir]); dbSeqDir = dir([rootDir biwiDir]);
if(depth)
output_dir = cat(2, output_dir, '_depth');
end
output_dir = cat(2, output_dir, '/'); output_dir = cat(2, output_dir, '/');
offset = 0; offset = 0;
@ -43,17 +39,12 @@ for i=3 + offset:numTogether:numel(dbSeqDir)
command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '"']); command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '"']);
if(depth)
dDir = [biwiDir dbSeqDir(i+n).name '/depthAligned/'];
command = cat(2, command, [' -fd "' dDir '"']);
end
if(verbose) if(verbose)
outputVideo = [output_dir dbSeqDir(i).name '.avi']; outputVideo = [output_dir dbSeqDir(i).name '.avi'];
command = cat(2, command, [' -ov "' outputVideo '"']); command = cat(2, command, [' -ov "' outputVideo '"']);
end end
end end
command = cat(2, command, [' -fx 505 -fy 505 -cx 320 -cy 240 -no2Dfp -no3Dfp -noMparams -noAUs -noGaze -vis-track']); command = cat(2, command, [' -fx 505 -fy 505 -cx 320 -cy 240 -no2Dfp -no3Dfp -noMparams -noAUs -noGaze -vis-track -camera_coord ']);
if(any(strcmp('model', varargin))) if(any(strcmp('model', varargin)))
command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']); command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']);

View file

@ -27,7 +27,7 @@ buDir = [database_root, '/bu/uniform-light/'];
% Run the Biwi test % Run the Biwi test
biwi_dir = '/biwi pose/'; biwi_dir = '/biwi pose/';
[res_folder_biwi_OF] = run_biwi_experiment(database_root, biwi_dir, false, false, 'model', 'model/main_clnf_general.txt'); [res_folder_biwi_OF] = run_biwi_experiment(database_root, biwi_dir, false, 'model', 'model/main_clnf_general.txt');
% Calculate the resulting errors % Calculate the resulting errors
[biwi_error_OF, pred_hp_biwi, gt_hp_biwi, ~, all_errors_biwi_OF, rels_biwi] = calcBiwiError(res_folder_biwi_OF, [database_root biwi_dir]); [biwi_error_OF, pred_hp_biwi, gt_hp_biwi, ~, all_errors_biwi_OF, rels_biwi] = calcBiwiError(res_folder_biwi_OF, [database_root biwi_dir]);
@ -35,7 +35,7 @@ biwi_dir = '/biwi pose/';
ict_dir = ['/ict/']; ict_dir = ['/ict/'];
% Intensity % Intensity
[res_folder_ict_OF] = run_ict_experiment(database_root, ict_dir, false, false, 'model', 'model/main_clnf_general.txt'); [res_folder_ict_OF] = run_ict_experiment(database_root, ict_dir, false, 'model', 'model/main_clnf_general.txt');
% Calculate the resulting errors % Calculate the resulting errors
[ict_error_OF, pred_hp_ict, gt_hp_ict, ~, all_errors_ict_OF, rel_ict] = calcIctError(res_folder_ict_OF, [database_root ict_dir]); [ict_error_OF, pred_hp_ict, gt_hp_ict, ~, all_errors_ict_OF, rel_ict] = calcIctError(res_folder_ict_OF, [database_root ict_dir]);

View file

@ -1,4 +1,4 @@
function [output_dir] = run_ict_experiment(rootDir, ictDir, verbose, depth, varargin) function [output_dir] = run_ict_experiment(rootDir, ictDir, verbose, varargin)
%EVALUATEICTDATABASE Summary of this function goes here %EVALUATEICTDATABASE Summary of this function goes here
% Detailed explanation goes here % Detailed explanation goes here
@ -12,10 +12,6 @@ output_dir = 'experiments/ict_out';
dbSeqDir = dir([rootDir ictDir]); dbSeqDir = dir([rootDir ictDir]);
if(depth)
output_dir = cat(2, output_dir, '_depth');
end
output_dir = cat(2, output_dir, '/'); output_dir = cat(2, output_dir, '/');
numTogether = 10; numTogether = 10;
@ -37,12 +33,7 @@ for i=3:numTogether:numel(dbSeqDir)
outputFile = [output_dir dbSeqDir(i+n).name '.txt']; outputFile = [output_dir dbSeqDir(i+n).name '.txt'];
command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '" ']); command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '" ']);
if(depth)
dDir = [ictDir dbSeqDir(i+n).name '/depthAligned/'];
command = cat(2, command, [' -fd "' dDir '"']);
end
if(verbose) if(verbose)
outputVideo = [output_dir dbSeqDir(i+n).name '.avi']; outputVideo = [output_dir dbSeqDir(i+n).name '.avi'];
command = cat(2, command, [' -ov "' outputVideo '"']); command = cat(2, command, [' -ov "' outputVideo '"']);