diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index e3d2ab6..4b94143 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -238,7 +238,6 @@ int main (int argc, char **argv) { // Reading the images - cv::Mat_ depth_image; cv::Mat_ grayscale_image; cv::Mat disp_image = captured_image.clone(); @@ -398,12 +397,6 @@ int main (int argc, char **argv) { cv::namedWindow("tracking_result",1); cv::imshow("tracking_result", disp_image); - - if(!depth_image.empty()) - { - // Division needed for visualisation purposes - imshow("depth", depth_image/2000.0); - } } // output the tracked video diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 071e605..4a5fe8b 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -232,7 +232,7 @@ int main (int argc, char **argv) // Get the input output file parameters // Indicates that rotation should be with respect to camera or world coordinates - bool use_camera_coordinates; + bool use_camera_coordinates = false; string output_codec; //not used but should LandmarkDetector::get_video_input_output_params(input_files, output_files, tracked_videos_output, use_camera_coordinates, output_codec, arguments); diff --git a/lib/local/LandmarkDetector/include/CCNF_patch_expert.h b/lib/local/LandmarkDetector/include/CCNF_patch_expert.h index c39af9a..e7482f9 100644 --- a/lib/local/LandmarkDetector/include/CCNF_patch_expert.h +++ b/lib/local/LandmarkDetector/include/CCNF_patch_expert.h @@ -51,7 +51,7 @@ class CCNF_neuron{ public: - // Type of patch (0=raw,1=grad,3=depth, other types besides raw are not actually used now) + // Type of patch (0=raw,1=grad other types besides raw are not actually used now) int neuron_type; // scaling of weights (needed as the energy of neuron might not be 1) @@ -112,7 +112,7 @@ public: void Read(std::ifstream &stream, std::vector window_sizes, std::vector > > sigma_components); - // actual work (can pass in an image and a potential depth image, if the CCNF is trained with depth) + // actual work (can pass in an image) void Response(cv::Mat_ &area_of_interest, cv::Mat_ &response); // Helper function to compute relevant sigmas diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index d4a360d..10d2c63 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -53,7 +53,7 @@ namespace LandmarkDetector // Helper functions for parsing the inputs //============================================================================================= void get_video_input_output_params(vector &input_video_file, vector &output_files, - vector &output_video_files, bool& world_coordinates_pose, string &output_codec, vector &arguments); + vector &output_video_files, bool& camera_coordinates_pose, string &output_codec, vector &arguments); void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector &arguments); diff --git a/lib/local/LandmarkDetector/include/SVR_patch_expert.h b/lib/local/LandmarkDetector/include/SVR_patch_expert.h index a762f73..d038d2f 100644 --- a/lib/local/LandmarkDetector/include/SVR_patch_expert.h +++ b/lib/local/LandmarkDetector/include/SVR_patch_expert.h @@ -78,9 +78,8 @@ class SVR_patch_expert{ // Reading in the patch expert void Read(std::ifstream &stream); - // The actual response computation from intensity or depth (for CLM-Z) + // The actual response computation from intensity void Response(const cv::Mat_ &area_of_interest, cv::Mat_ &response); - void ResponseDepth(const cv::Mat_ &area_of_interest, cv::Mat_ &response); }; //=========================================================================== @@ -105,9 +104,8 @@ class Multi_SVR_patch_expert{ void Read(std::ifstream &stream); - // actual response computation from intensity of depth (for CLM-Z) + // actual response computation from intensity void Response(const cv::Mat_ &area_of_interest, cv::Mat_ &response); - void ResponseDepth(const cv::Mat_ &area_of_interest, cv::Mat_ &response); }; } diff --git a/lib/local/LandmarkDetector/src/SVR_patch_expert.cpp b/lib/local/LandmarkDetector/src/SVR_patch_expert.cpp index 6a01f10..66293e8 100644 --- a/lib/local/LandmarkDetector/src/SVR_patch_expert.cpp +++ b/lib/local/LandmarkDetector/src/SVR_patch_expert.cpp @@ -195,72 +195,6 @@ void SVR_patch_expert::Response(const cv::Mat_& area_of_interest, cv::Mat } -void SVR_patch_expert::ResponseDepth(const cv::Mat_& area_of_interest, cv::Mat_ &response) -{ - - // How big the response map will be - int response_height = area_of_interest.rows - weights.rows + 1; - int response_width = area_of_interest.cols - weights.cols + 1; - - // the patch area on which we will calculate reponses - cv::Mat_ normalised_area_of_interest; - - if(response.rows != response_height || response.cols != response_width) - { - response.create(response_height, response_width); - } - - if(type == 0) - { - // Perform normalisation across whole patch - cv::Scalar mean; - cv::Scalar std; - - // ignore missing values - cv::Mat_ mask = area_of_interest > 0; - cv::meanStdDev(area_of_interest, mean, std, mask); - - // if all values the same don't divide by 0 - if(std[0] == 0) - { - std[0] = 1; - } - - normalised_area_of_interest = (area_of_interest - mean[0]) / std[0]; - - // Set the invalid pixels to 0 - normalised_area_of_interest.setTo(0, mask == 0); - } - else - { - printf("ERROR(%s,%d): Unsupported patch type %d!\n", __FILE__,__LINE__,type); - abort(); - } - - cv::Mat_ svr_response; - - // The empty matrix as we don't pass precomputed dft's of image - cv::Mat_ empty_matrix_0(0,0,0.0); - cv::Mat_ empty_matrix_1(0,0,0.0); - cv::Mat_ empty_matrix_2(0,0,0.0); - - // Efficient calc of patch expert response across the area of interest - - matchTemplate_m(normalised_area_of_interest, empty_matrix_0, empty_matrix_1, empty_matrix_2, weights, weights_dfts, svr_response, CV_TM_CCOEFF); - - response.create(svr_response.size()); - cv::MatIterator_ p = response.begin(); - - cv::MatIterator_ q1 = svr_response.begin(); // respone for each pixel - cv::MatIterator_ q2 = svr_response.end(); - - while(q1 != q2) - { - // the SVR response passed through a logistic regressor - *p++ = 1.0/(1.0 + exp( -(*q1++ * scaling + bias ))); - } -} - // Copy constructor Multi_SVR_patch_expert::Multi_SVR_patch_expert(const Multi_SVR_patch_expert& other) : svr_patch_experts(other.svr_patch_experts) { @@ -321,17 +255,3 @@ void Multi_SVR_patch_expert::Response(const cv::Mat_ &area_of_interest, c } -void Multi_SVR_patch_expert::ResponseDepth(const cv::Mat_& area_of_interest, cv::Mat_& response) -{ - int response_height = area_of_interest.rows - height + 1; - int response_width = area_of_interest.cols - width + 1; - - if(response.rows != response_height || response.cols != response_width) - { - response.create(response_height, response_width); - } - - // With depth patch experts only do raw data modality - svr_patch_experts[0].ResponseDepth(area_of_interest, response); -} -//=========================================================================== diff --git a/matlab_runners/Feature Point Experiments/results/fps_yt.mat b/matlab_runners/Feature Point Experiments/results/fps_yt.mat index 37cae7b..b4b90a5 100644 Binary files a/matlab_runners/Feature Point Experiments/results/fps_yt.mat and b/matlab_runners/Feature Point Experiments/results/fps_yt.mat differ diff --git a/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf b/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf index 74eaf56..7e88056 100644 Binary files a/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf and b/matlab_runners/Feature Point Experiments/results/in-the-wild-res-no-outline.pdf differ diff --git a/matlab_runners/Feature Point Experiments/results/landmark_detections.mat b/matlab_runners/Feature Point Experiments/results/landmark_detections.mat index 6c78696..826d2ce 100644 Binary files a/matlab_runners/Feature Point Experiments/results/landmark_detections.mat and b/matlab_runners/Feature Point Experiments/results/landmark_detections.mat differ diff --git a/matlab_runners/Gaze Experiments/mpii_1500_errs.mat b/matlab_runners/Gaze Experiments/mpii_1500_errs.mat index 59c3df6..8fa2d5d 100644 Binary files a/matlab_runners/Gaze Experiments/mpii_1500_errs.mat and b/matlab_runners/Gaze Experiments/mpii_1500_errs.mat differ diff --git a/matlab_runners/Gaze Experiments/mpii_1500_errs.txt b/matlab_runners/Gaze Experiments/mpii_1500_errs.txt index 4058ebd..d85ed47 100644 --- a/matlab_runners/Gaze Experiments/mpii_1500_errs.txt +++ b/matlab_runners/Gaze Experiments/mpii_1500_errs.txt @@ -1,2 +1,2 @@ Mean error, median error -9.469, 8.773 +9.405, 8.580 diff --git a/matlab_runners/Head Pose Experiments/results/Pose_OF.mat b/matlab_runners/Head Pose Experiments/results/Pose_OF.mat index 74a248e..8e6a34d 100644 Binary files a/matlab_runners/Head Pose Experiments/results/Pose_OF.mat and b/matlab_runners/Head Pose Experiments/results/Pose_OF.mat differ diff --git a/matlab_runners/Head Pose Experiments/results/Pose_OF.txt b/matlab_runners/Head Pose Experiments/results/Pose_OF.txt index 36f94a1..e5ce8be 100644 --- a/matlab_runners/Head Pose Experiments/results/Pose_OF.txt +++ b/matlab_runners/Head Pose Experiments/results/Pose_OF.txt @@ -1,4 +1,4 @@ Dataset and model, pitch, yaw, roll, mean, median biwi error: 7.092, 5.170, 4.657, 5.640, 2.607 -bu error: 2.769, 4.105, 2.569, 3.147, 2.118 -ict error: 3.489, 3.632, 3.538, 3.553, 2.029 +bu error: 2.762, 3.328, 2.249, 2.780, 1.963 +ict error: 3.529, 3.873, 2.688, 3.363, 1.931 diff --git a/matlab_runners/Head Pose Experiments/run_biwi_experiment.m b/matlab_runners/Head Pose Experiments/run_biwi_experiment.m index d093c10..cdf7f0f 100644 --- a/matlab_runners/Head Pose Experiments/run_biwi_experiment.m +++ b/matlab_runners/Head Pose Experiments/run_biwi_experiment.m @@ -1,4 +1,4 @@ -function [output_dir] = run_biwi_experiment(rootDir, biwiDir, verbose, depth, varargin) +function [output_dir] = run_biwi_experiment(rootDir, biwiDir, verbose, varargin) % Biwi dataset experiment if(isunix) @@ -11,10 +11,6 @@ output_dir = 'experiments/biwi_out'; dbSeqDir = dir([rootDir biwiDir]); -if(depth) - output_dir = cat(2, output_dir, '_depth'); -end - output_dir = cat(2, output_dir, '/'); offset = 0; @@ -43,17 +39,12 @@ for i=3 + offset:numTogether:numel(dbSeqDir) command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '"']); - if(depth) - dDir = [biwiDir dbSeqDir(i+n).name '/depthAligned/']; - command = cat(2, command, [' -fd "' dDir '"']); - end - if(verbose) outputVideo = [output_dir dbSeqDir(i).name '.avi']; command = cat(2, command, [' -ov "' outputVideo '"']); end end - command = cat(2, command, [' -fx 505 -fy 505 -cx 320 -cy 240 -no2Dfp -no3Dfp -noMparams -noAUs -noGaze -vis-track']); + command = cat(2, command, [' -fx 505 -fy 505 -cx 320 -cy 240 -no2Dfp -no3Dfp -noMparams -noAUs -noGaze -vis-track -camera_coord ']); if(any(strcmp('model', varargin))) command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']); diff --git a/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m b/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m index 7059f13..f22ddce 100644 --- a/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m +++ b/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m @@ -27,7 +27,7 @@ buDir = [database_root, '/bu/uniform-light/']; % Run the Biwi test biwi_dir = '/biwi pose/'; -[res_folder_biwi_OF] = run_biwi_experiment(database_root, biwi_dir, false, false, 'model', 'model/main_clnf_general.txt'); +[res_folder_biwi_OF] = run_biwi_experiment(database_root, biwi_dir, false, 'model', 'model/main_clnf_general.txt'); % Calculate the resulting errors [biwi_error_OF, pred_hp_biwi, gt_hp_biwi, ~, all_errors_biwi_OF, rels_biwi] = calcBiwiError(res_folder_biwi_OF, [database_root biwi_dir]); @@ -35,7 +35,7 @@ biwi_dir = '/biwi pose/'; ict_dir = ['/ict/']; % Intensity -[res_folder_ict_OF] = run_ict_experiment(database_root, ict_dir, false, false, 'model', 'model/main_clnf_general.txt'); +[res_folder_ict_OF] = run_ict_experiment(database_root, ict_dir, false, 'model', 'model/main_clnf_general.txt'); % Calculate the resulting errors [ict_error_OF, pred_hp_ict, gt_hp_ict, ~, all_errors_ict_OF, rel_ict] = calcIctError(res_folder_ict_OF, [database_root ict_dir]); diff --git a/matlab_runners/Head Pose Experiments/run_ict_experiment.m b/matlab_runners/Head Pose Experiments/run_ict_experiment.m index 5e23942..75e2352 100644 --- a/matlab_runners/Head Pose Experiments/run_ict_experiment.m +++ b/matlab_runners/Head Pose Experiments/run_ict_experiment.m @@ -1,4 +1,4 @@ -function [output_dir] = run_ict_experiment(rootDir, ictDir, verbose, depth, varargin) +function [output_dir] = run_ict_experiment(rootDir, ictDir, verbose, varargin) %EVALUATEICTDATABASE Summary of this function goes here % Detailed explanation goes here @@ -12,10 +12,6 @@ output_dir = 'experiments/ict_out'; dbSeqDir = dir([rootDir ictDir]); -if(depth) - output_dir = cat(2, output_dir, '_depth'); -end - output_dir = cat(2, output_dir, '/'); numTogether = 10; @@ -37,12 +33,7 @@ for i=3:numTogether:numel(dbSeqDir) outputFile = [output_dir dbSeqDir(i+n).name '.txt']; command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '" ']); - - if(depth) - dDir = [ictDir dbSeqDir(i+n).name '/depthAligned/']; - command = cat(2, command, [' -fd "' dDir '"']); - end - + if(verbose) outputVideo = [output_dir dbSeqDir(i+n).name '.avi']; command = cat(2, command, [' -ov "' outputVideo '"']);