diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index ced96ac..9e07b2e 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -314,10 +314,6 @@ int main (int argc, char **argv) //Convert arguments to more convenient vector form vector arguments = get_arguments(argc, argv); - // Search paths - boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); - boost::filesystem::path parent_path = boost::filesystem::path(arguments[0]).parent_path(); - // Some initial parameters that can be overriden from command line vector files, output_images, output_landmark_locations, output_pose_locations; @@ -354,50 +350,10 @@ int main (int argc, char **argv) cv::CascadeClassifier classifier(det_parameters.face_detector_location); dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector(); - // Loading the AU prediction models - string au_loc = "AU_predictors/AU_all_static.txt"; - - boost::filesystem::path au_loc_path = boost::filesystem::path(au_loc); - if (boost::filesystem::exists(au_loc_path)) - { - au_loc = au_loc_path.string(); - } - else if (boost::filesystem::exists(parent_path/au_loc_path)) - { - au_loc = (parent_path/au_loc_path).string(); - } - else if (boost::filesystem::exists(config_path/au_loc_path)) - { - au_loc = (config_path/au_loc_path).string(); - } - else - { - cout << "Can't find AU prediction files, exiting" << endl; - return 1; - } - - // Used for image masking for AUs - string tri_loc; - boost::filesystem::path tri_loc_path = boost::filesystem::path("model/tris_68_full.txt"); - if (boost::filesystem::exists(tri_loc_path)) - { - tri_loc = tri_loc_path.string(); - } - else if (boost::filesystem::exists(parent_path/tri_loc_path)) - { - tri_loc = (parent_path/tri_loc_path).string(); - } - else if (boost::filesystem::exists(config_path/tri_loc_path)) - { - tri_loc = (config_path/tri_loc_path).string(); - } - else - { - cout << "Can't find triangulation files, exiting" << endl; - return 1; - } - - FaceAnalysis::FaceAnalyser face_analyser(vector(), 0.7, 112, 112, au_loc, tri_loc); + // Load facial feature extractor and AU analyser (make sure it is static) + FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); + face_analysis_params.OptimizeForImages(); + FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); bool visualise = !det_parameters.quiet_mode; @@ -477,7 +433,7 @@ int main (int argc, char **argv) gazeAngle = FaceAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); } - auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false); + auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model.detected_landmarks, false); // Writing out the detected landmarks (in an OS independent manner) if(!output_landmark_locations.empty()) @@ -594,7 +550,7 @@ int main (int argc, char **argv) gazeAngle = FaceAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); } - auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false); + auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model.detected_landmarks, false); // Writing out the detected landmarks if(!output_landmark_locations.empty()) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index c13a7a6..2d9bc38 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -131,9 +131,9 @@ void create_directory(string output_path) } } -void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, double &similarity_scale, - int &similarity_size, bool &grayscale, bool& visualize_track, bool& visualize_align, bool& visualize_hog, bool& dynamic, bool &output_2D_landmarks, bool &output_3D_landmarks, - bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze, vector &arguments); +void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, bool& visualize_track, + bool& visualize_align, bool& visualize_hog, bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, + bool &output_pose, bool &output_AUs, bool &output_gaze, vector &arguments); void get_image_input_output_params_feats(vector > &input_image_files, bool& as_video, vector &arguments); @@ -217,18 +217,10 @@ int main (int argc, char **argv) { vector arguments = get_arguments(argc, argv); - - // Search paths - boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); - boost::filesystem::path parent_path = boost::filesystem::path(arguments[0]).parent_path(); - + // Some initial parameters that can be overriden from command line vector input_files, output_files, tracked_videos_output; - LandmarkDetector::FaceModelParameters det_parameters(arguments); - // Always track gaze in feature extraction - det_parameters.track_gaze = true; - // Get the input output file parameters // Indicates that rotation should be with respect to camera or world coordinates @@ -274,20 +266,9 @@ int main (int argc, char **argv) fx_undefined = true; } - // The modules that are being used for tracking - LandmarkDetector::CLNF face_model(det_parameters.model_location); - vector output_similarity_align; vector output_hog_align_files; - double sim_scale = -1; - int sim_size = 112; - bool grayscale = false; - bool video_output = false; - bool dynamic = true; // Indicates if a dynamic AU model should be used (dynamic is useful if the video is long enough to include neutral expressions) - int num_hog_rows; - int num_hog_cols; - // By default output all parameters, but these can be turned off to get smaller files or slightly faster processing times // use -no2Dfp, -no3Dfp, -noMparams, -noPose, -noAUs, -noGaze to turn them off bool output_2D_landmarks = true; @@ -300,77 +281,25 @@ int main (int argc, char **argv) bool visualize_track = false; bool visualize_align = false; bool visualize_hog = false; - get_output_feature_params(output_similarity_align, output_hog_align_files, sim_scale, sim_size, grayscale, visualize_track, visualize_align, visualize_hog, dynamic, + get_output_feature_params(output_similarity_align, output_hog_align_files, visualize_track, visualize_align, visualize_hog, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, arguments); - // Used for image masking - string tri_loc; - boost::filesystem::path tri_loc_path = boost::filesystem::path("model/tris_68_full.txt"); - if (boost::filesystem::exists(tri_loc_path)) - { - tri_loc = tri_loc_path.string(); - } - else if (boost::filesystem::exists(parent_path/tri_loc_path)) - { - tri_loc = (parent_path/tri_loc_path).string(); - } - else if (boost::filesystem::exists(config_path/tri_loc_path)) - { - tri_loc = (config_path/tri_loc_path).string(); - } - else - { - cout << "Can't find triangulation files, exiting" << endl; - return 1; - } - - // Will warp to scaled mean shape - cv::Mat_ similarity_normalised_shape = face_model.pdm.mean_shape * sim_scale; - // Discard the z component - similarity_normalised_shape = similarity_normalised_shape(cv::Rect(0, 0, 1, 2*similarity_normalised_shape.rows/3)).clone(); - // If multiple video files are tracked, use this to indicate if we are done bool done = false; int f_n = -1; int curr_img = -1; - string au_loc; + // Load the modules that are being used for tracking and face analysis + // Load face landmark detector + LandmarkDetector::FaceModelParameters det_parameters(arguments); + // Always track gaze in feature extraction + det_parameters.track_gaze = true; + LandmarkDetector::CLNF face_model(det_parameters.model_location); - string au_loc_local; - if (dynamic) - { - au_loc_local = "AU_predictors/AU_all_best.txt"; - } - else - { - au_loc_local = "AU_predictors/AU_all_static.txt"; - } + // Load facial feature extractor and AU analyser + FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); + FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); - boost::filesystem::path au_loc_path = boost::filesystem::path(au_loc_local); - if (boost::filesystem::exists(au_loc_path)) - { - au_loc = au_loc_path.string(); - } - else if (boost::filesystem::exists(parent_path/au_loc_path)) - { - au_loc = (parent_path/au_loc_path).string(); - } - else if (boost::filesystem::exists(config_path/au_loc_path)) - { - au_loc = (config_path/au_loc_path).string(); - } - else - { - cout << "Can't find AU prediction files, exiting" << endl; - return 1; - } - - // Creating a face analyser that will be used for AU extraction - // Make sure sim_scale is proportional to sim_size if not set - if (sim_scale == -1) sim_scale = sim_size * (0.7 / 112.0); - - FaceAnalysis::FaceAnalyser face_analyser(vector(), sim_scale, sim_size, sim_size, au_loc, tri_loc); - while(!done) // this is not a for loop as we might also be reading from a webcam { @@ -562,11 +491,12 @@ int main (int argc, char **argv) // Do face alignment cv::Mat sim_warped_img; cv::Mat_ hog_descriptor; + int num_hog_rows, num_hog_cols; // But only if needed in output if(!output_similarity_align.empty() || hog_output_file.is_open() || output_AUs) { - face_analyser.AddNextFrame(captured_image, face_model, time_stamp, false, !det_parameters.quiet_mode && (visualize_align || visualize_hog)); + face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, time_stamp, false, !det_parameters.quiet_mode); face_analyser.GetLatestAlignedFace(sim_warped_img); if(!det_parameters.quiet_mode && visualize_align) @@ -606,11 +536,6 @@ int main (int argc, char **argv) if (!output_similarity_align.empty()) { - if (sim_warped_img.channels() == 3 && grayscale) - { - cvtColor(sim_warped_img, sim_warped_img, CV_BGR2GRAY); - } - char name[100]; // Filename is based on frame number @@ -702,7 +627,7 @@ int main (int argc, char **argv) if (output_files.size() > 0 && output_AUs) { cout << "Postprocessing the Action Unit predictions" << endl; - face_analyser.PostprocessOutputFile(output_files[f_n], dynamic); + face_analyser.PostprocessOutputFile(output_files[f_n]); } // Reset the models for the next video face_analyser.Reset(); @@ -976,10 +901,9 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo } -void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, double &similarity_scale, - int &similarity_size, bool &grayscale, bool& visualize_track, bool& visualize_align, bool& visualize_hog, bool& dynamic, - bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze, - vector &arguments) +void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, bool& visualize_track, + bool& visualize_align, bool& visualize_hog, bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, + bool &output_pose, bool &output_AUs, bool &output_gaze, vector &arguments) { output_similarity_aligned.clear(); output_hog_aligned_files.clear(); @@ -993,9 +917,6 @@ void get_output_feature_params(vector &output_similarity_aligned, vector string output_root = ""; - // By default the model is dynamic - dynamic = true; - visualize_align = false; visualize_hog = false; visualize_track = false; @@ -1056,29 +977,6 @@ void get_output_feature_params(vector &output_similarity_aligned, vector visualize_track = true; valid[i] = false; } - else if (arguments[i].compare("-au_static") == 0) - { - dynamic = false; - } - else if (arguments[i].compare("-g") == 0) - { - grayscale = true; - valid[i] = false; - } - else if (arguments[i].compare("-simscale") == 0) - { - similarity_scale = stod(arguments[i + 1]); - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-simsize") == 0) - { - similarity_size = stoi(arguments[i + 1]); - valid[i] = false; - valid[i + 1] = false; - i++; - } else if (arguments[i].compare("-no2Dfp") == 0) { output_2D_landmarks = false; diff --git a/lib/local/FaceAnalyser/AU_predictors/In-the-wild_aligned_PDM_68.txt b/lib/local/FaceAnalyser/AU_predictors/In-the-wild_aligned_PDM_68.txt new file mode 100644 index 0000000..d538397 --- /dev/null +++ b/lib/local/FaceAnalyser/AU_predictors/In-the-wild_aligned_PDM_68.txt @@ -0,0 +1,421 @@ +# The mean values of the components (in mm) +204 +1 +6 +-73.393523 +-72.775014 +-70.533638 +-66.850058 +-59.790187 +-48.368973 +-34.121101 +-17.875411 +0.098749 +17.477031 +32.648966 +46.372358 +57.343480 +64.388482 +68.212038 +70.486405 +71.375822 +-61.119406 +-51.287588 +-37.804800 +-24.022754 +-11.635713 +12.056636 +25.106256 +38.338588 +51.191007 +60.053851 +0.653940 +0.804809 +0.992204 +1.226783 +-14.772472 +-7.180239 +0.555920 +8.272499 +15.214351 +-46.047290 +-37.674688 +-27.883856 +-19.648268 +-28.272965 +-38.082418 +19.265868 +27.894191 +37.437529 +45.170805 +38.196454 +28.764989 +-28.916267 +-17.533194 +-6.684590 +0.381001 +8.375443 +18.876618 +28.794412 +19.057574 +8.956375 +0.381549 +-7.428895 +-18.160634 +-24.377490 +-6.897633 +0.340663 +8.444722 +24.474473 +8.449166 +0.205322 +-7.198266 +-29.801432 +-10.949766 +7.929818 +26.074280 +42.564390 +56.481080 +67.246992 +75.056892 +77.061286 +74.758448 +66.929021 +56.311389 +42.419126 +25.455880 +6.990805 +-11.666193 +-30.365191 +-49.361602 +-58.769795 +-61.996155 +-61.033399 +-56.686759 +-57.391033 +-61.902186 +-62.777713 +-59.302347 +-50.190255 +-42.193790 +-30.993721 +-19.944596 +-8.414541 +2.598255 +4.751589 +6.562900 +4.661005 +2.643046 +-37.471411 +-42.730510 +-42.711517 +-36.754742 +-35.134493 +-34.919043 +-37.032306 +-43.342445 +-43.110822 +-38.086515 +-35.532024 +-35.484289 +28.612716 +22.172187 +19.029051 +20.721118 +19.035460 +22.394109 +28.079924 +36.298248 +39.634575 +40.395647 +39.836405 +36.677899 +28.677771 +25.475976 +26.014269 +25.326198 +28.323008 +30.596216 +31.408738 +30.844876 +47.667532 +45.909403 +44.842580 +43.141114 +38.635298 +30.750622 +18.456453 +3.609035 +-0.881698 +5.181201 +19.176563 +30.770570 +37.628629 +40.886309 +42.281449 +44.142567 +47.140426 +14.254422 +7.268147 +0.442051 +-6.606501 +-11.967398 +-12.051204 +-7.315098 +-1.022953 +5.349435 +11.615746 +-13.380835 +-21.150853 +-29.284036 +-36.948060 +-20.132003 +-23.536684 +-25.944448 +-23.695741 +-20.858157 +7.037989 +3.021217 +1.353629 +-0.111088 +-0.147273 +1.476612 +-0.665746 +0.247660 +1.696435 +4.894163 +0.282961 +-1.172675 +-2.240310 +-15.934335 +-22.611355 +-23.748437 +-22.721995 +-15.610679 +-3.217393 +-14.987997 +-22.554245 +-23.591626 +-22.406106 +-15.121907 +-4.785684 +-20.893742 +-22.220479 +-21.025520 +-5.712776 +-20.671489 +-21.903670 +-20.328022 +# The principal components (eigenvectors) of identity or combined identity and expression model +204 +34 +6 +-0.007395 -0.093690 0.039362 0.204443 -0.104698 0.033568 -0.090173 0.010170 -0.042341 0.104375 0.032695 0.038750 0.064385 -0.037025 0.058377 0.032457 -0.100005 -0.082042 -0.053440 0.008782 -0.027164 -0.000368 -0.129614 0.035436 -0.062685 -0.075349 0.140764 -0.032290 -0.115829 -0.037865 0.068590 0.008886 -0.066442 0.259211 +-0.002553 -0.075344 0.037127 0.201967 -0.069124 0.041645 -0.100939 0.018352 0.006598 0.081569 0.004894 0.038886 0.050631 0.056656 0.055353 -0.028457 -0.022518 -0.089693 -0.079196 0.010851 -0.017583 0.015095 -0.077834 -0.000944 -0.104255 -0.060338 0.106169 -0.065799 -0.067068 -0.051588 0.080490 0.063152 -0.047531 0.178216 +-0.002097 -0.055385 0.025705 0.215785 -0.026274 0.053548 -0.086364 0.041926 0.035784 0.033485 -0.014602 0.049999 0.031950 0.125201 0.045372 -0.064200 0.070073 -0.074313 -0.083942 -0.007566 -0.003666 0.034226 -0.018354 -0.011639 -0.084633 -0.039431 0.064057 -0.056515 -0.046926 -0.074699 0.073477 0.074109 -0.054882 0.046675 +-0.005533 -0.035821 0.019787 0.226957 0.030415 0.059508 -0.051058 0.057386 0.023240 -0.025487 -0.013088 0.043330 0.025141 0.113119 0.017370 -0.071064 0.108140 -0.049138 -0.043417 -0.065375 0.012959 0.070483 0.075536 -0.035685 -0.005847 0.012618 0.040297 -0.049743 -0.061354 -0.087164 0.033762 0.048771 -0.096563 -0.045189 +-0.006914 0.011987 0.030022 0.205560 0.099633 0.072384 -0.008313 0.060244 -0.006584 -0.104943 -0.018036 0.032806 0.029879 0.068434 -0.043721 -0.052564 0.077505 -0.062192 0.026865 -0.122834 -0.001529 0.064443 0.160557 -0.036750 0.124476 0.072641 0.048609 -0.043443 -0.060729 -0.040443 -0.050540 0.050509 -0.118886 -0.146867 +-0.004182 0.064263 0.036886 0.149341 0.158261 0.075501 0.030568 0.067074 -0.040268 -0.167116 -0.020230 0.029360 0.046726 0.033410 -0.121685 -0.004528 -0.017505 -0.050390 0.063237 -0.133203 -0.070794 0.005133 0.178032 -0.002768 0.153446 0.105966 0.061693 0.011010 -0.027442 0.001944 -0.103967 0.045708 -0.083197 -0.172889 +0.016954 0.088125 0.034199 0.081242 0.170872 0.048964 0.063434 0.083842 -0.058188 -0.164769 -0.012135 0.002128 0.084787 -0.009691 -0.136231 0.068411 -0.091616 0.042046 0.053339 -0.068859 -0.141834 -0.040710 0.017020 0.034007 0.053187 0.096749 0.068416 0.075079 0.038728 0.063588 -0.109581 0.007737 0.033105 -0.152075 +0.021733 0.081042 0.015310 0.029492 0.112212 0.012959 0.065003 0.095565 -0.047228 -0.120559 -0.022904 -0.051461 0.036411 -0.036990 -0.101765 0.150918 -0.102123 0.163128 0.047199 0.037063 -0.223718 -0.065303 -0.149191 -0.017823 -0.134138 0.084419 0.010718 0.087970 0.043030 0.040099 -0.068487 -0.008773 0.077035 -0.082281 +0.024713 0.084952 0.002319 -0.002792 0.028194 -0.016856 0.036083 0.087381 -0.020743 -0.024386 -0.027268 -0.091654 -0.009025 -0.052124 -0.033407 0.168353 -0.061447 0.244847 0.024587 0.064221 -0.161138 0.020993 -0.164584 -0.068179 -0.181775 0.087509 -0.068202 0.059078 -0.004250 -0.001472 -0.031248 -0.049011 -0.019415 0.022163 +0.025404 0.095704 -0.013447 -0.036864 -0.064015 -0.035962 0.001144 0.050101 0.006172 0.084003 -0.015860 -0.102295 -0.028308 -0.053076 0.017520 0.110370 -0.009494 0.248830 -0.017769 0.064655 -0.062487 0.133872 -0.077193 -0.077102 -0.104610 0.084923 -0.116710 -0.000424 -0.001453 -0.001504 -0.004986 -0.072792 -0.151917 0.063339 +0.027234 0.088446 -0.037011 -0.090962 -0.124015 -0.052844 -0.020061 0.017974 0.031824 0.138013 -0.009523 -0.093892 -0.061812 -0.016350 0.043025 0.034106 0.036403 0.177560 -0.066137 0.039288 0.041972 0.145398 -0.004524 -0.021113 0.025045 0.065780 -0.102128 -0.061053 0.010043 -0.018688 0.018733 -0.002990 -0.240304 0.007274 +0.033301 0.051083 -0.050161 -0.137239 -0.130410 -0.078352 -0.030467 -0.016581 0.042916 0.153230 -0.008295 -0.048519 -0.056020 -0.017449 0.051644 -0.021078 0.049456 0.042549 -0.075780 0.051205 0.057200 0.117965 0.058462 0.078890 0.121853 0.027311 -0.037081 -0.082100 0.013948 0.019188 0.029703 0.076102 -0.243643 -0.067711 +0.034561 -0.024817 -0.037181 -0.171731 -0.091823 -0.080914 -0.018022 -0.019280 0.055534 0.112038 -0.006388 0.000380 -0.027767 -0.031689 0.027915 -0.012644 0.004351 -0.072270 -0.053990 0.028620 0.011428 0.077628 0.073278 0.159948 0.096141 0.006855 0.036568 -0.013290 -0.001587 0.033748 0.052795 0.164287 -0.166678 -0.098685 +0.031311 -0.094373 -0.015266 -0.181991 -0.032285 -0.067979 0.020822 -0.028457 0.050194 0.060652 0.006169 0.060618 0.005717 -0.047184 0.006512 0.012425 -0.059877 -0.141325 -0.029145 -0.010906 -0.069894 0.032082 0.054382 0.151384 0.041519 0.026929 0.076049 0.067107 -0.027885 0.040313 0.041582 0.194576 -0.086531 -0.080338 +0.030327 -0.127857 -0.008012 -0.166922 0.025066 -0.062660 0.063753 -0.029802 0.044933 0.028298 0.025152 0.109179 0.013585 -0.043385 0.003918 0.007538 -0.080941 -0.141969 0.011067 -0.055727 -0.099322 -0.018025 0.009874 0.065510 -0.019589 0.084273 0.059580 0.099232 -0.026102 0.043042 0.023322 0.151827 -0.031702 -0.023000 +0.030239 -0.141766 -0.020490 -0.154724 0.073176 -0.066098 0.076339 -0.045120 0.049403 -0.014474 0.024782 0.109864 0.005943 -0.008921 0.001482 -0.008304 -0.060453 -0.100507 0.030769 -0.093919 -0.042422 -0.045051 -0.014060 -0.020183 -0.093279 0.134797 0.006048 0.040717 -0.038709 0.039630 0.006137 0.076303 0.021244 0.014680 +0.030493 -0.158938 -0.031460 -0.156567 0.120023 -0.070626 0.084879 -0.065759 0.063746 -0.047728 -0.002344 0.094855 -0.013995 0.014159 0.011125 -0.034394 -0.064702 -0.055871 0.041269 -0.110361 0.045722 -0.064289 -0.013213 -0.104140 -0.133226 0.183607 -0.037284 0.002311 -0.041808 0.033510 -0.034315 0.012710 0.065609 0.046794 +-0.029307 -0.000612 0.006003 0.066897 -0.126329 0.008282 0.106455 -0.058070 -0.127455 0.112217 0.028721 -0.008227 0.023123 -0.184660 -0.000131 0.060814 -0.038192 0.061802 0.055924 -0.086767 -0.011372 0.116214 0.039681 -0.047083 0.046626 0.042705 0.011194 -0.002237 0.023832 0.021378 0.074731 -0.040608 -0.025003 0.203421 +-0.038458 0.020813 -0.004500 0.021759 -0.086203 0.017004 0.123118 -0.051398 -0.154096 0.048594 0.160783 0.035847 -0.105209 -0.131283 -0.034334 0.088202 -0.022053 0.065794 0.043514 -0.164773 0.038562 0.221823 -0.008980 0.076251 -0.067806 0.014541 0.027230 -0.014379 -0.016754 -0.010835 0.026126 -0.010414 0.029183 0.058614 +-0.021490 0.036216 -0.005865 0.020024 -0.060804 0.026512 0.126241 -0.083197 -0.120419 0.032472 0.187630 -0.015555 -0.159327 -0.011903 -0.087082 0.079345 -0.061540 0.015194 0.062573 -0.039469 0.032277 0.191416 0.026454 0.082398 -0.056523 -0.017131 0.027510 -0.008610 0.004494 -0.034043 -0.017450 0.006765 0.078634 -0.036122 +0.000867 0.042636 0.003104 0.029552 -0.036234 0.016402 0.123844 -0.126971 -0.075012 0.060293 0.148833 -0.081953 -0.173218 0.088994 -0.136465 0.063644 -0.092909 -0.034989 0.083554 0.107321 0.022452 0.121649 0.095422 0.064093 -0.044127 -0.041514 -0.001734 -0.007934 0.040275 -0.042178 -0.037676 0.021143 0.091871 -0.056272 +0.024120 0.058528 0.006018 0.028610 -0.012243 0.008813 0.097121 -0.159602 -0.017056 0.077811 0.087002 -0.120528 -0.150197 0.184860 -0.180434 0.040369 -0.097894 -0.042266 0.075094 0.203623 0.039492 0.033806 0.132455 0.059185 -0.102173 -0.071020 -0.051542 0.002257 0.073697 -0.021845 -0.035627 0.009661 0.123071 -0.065039 +-0.056420 0.059125 -0.030321 -0.057038 0.055493 -0.007116 -0.126529 0.019949 -0.058614 -0.078223 0.002145 0.050900 0.167834 -0.173333 0.181067 -0.004047 0.063488 0.151506 -0.125850 -0.136067 0.023547 0.034287 -0.056834 0.003824 0.134406 -0.055792 0.087938 0.068804 -0.033240 0.076407 -0.033551 -0.097730 0.084249 -0.079917 +-0.038591 0.041658 -0.021071 -0.054808 0.085131 -0.020672 -0.150923 -0.028119 -0.005051 -0.082123 -0.027470 0.004437 0.151800 -0.139134 0.205779 0.055682 0.024516 0.096154 -0.038417 -0.113849 0.045637 0.075165 -0.039525 0.011140 0.058496 -0.061165 0.070389 0.031381 -0.020956 0.065917 -0.033395 -0.096762 0.067056 0.005758 +-0.007940 0.026461 -0.003035 -0.037585 0.112114 -0.032320 -0.154598 -0.070183 0.034104 -0.066949 -0.048103 -0.076036 0.108505 -0.081056 0.189128 0.100712 0.001290 0.009722 0.046728 -0.023127 0.013390 0.076873 0.012671 0.031037 0.075639 -0.064473 0.061414 -0.019564 0.014597 0.046786 -0.011485 -0.084927 -0.000334 0.061175 +0.018946 -0.000145 0.011179 -0.029672 0.136914 -0.030086 -0.147824 -0.105721 0.066935 -0.079902 -0.036968 -0.110909 0.052283 0.004188 0.126896 0.134626 -0.006676 -0.052490 0.091685 0.063396 -0.013036 0.002970 0.053049 0.019284 0.038579 -0.025497 0.025340 -0.089926 0.050450 -0.006666 -0.013118 -0.030505 -0.058633 0.064143 +0.026629 -0.059165 -0.000823 -0.058599 0.151849 -0.022073 -0.130501 -0.084385 0.071751 -0.134872 0.039773 -0.057794 -0.044027 0.076494 0.035989 0.101529 0.005315 -0.029779 0.032498 0.028803 0.005618 -0.063529 -0.004924 -0.022183 -0.113427 0.055922 -0.008186 -0.140830 0.090960 -0.114112 -0.009417 0.083947 -0.041032 -0.006371 +-0.010951 0.075350 -0.001817 -0.012876 0.013096 0.004579 -0.016153 -0.047823 -0.029850 0.016150 0.033334 0.015682 0.035621 0.037191 -0.019504 -0.019256 0.044839 0.020295 -0.073131 0.039780 -0.036879 -0.040970 0.053655 -0.001482 -0.002821 -0.032902 -0.000066 0.005729 0.001116 0.056517 -0.017058 -0.021947 0.037761 -0.039276 +-0.012100 0.064207 -0.001096 -0.013941 0.006814 0.008710 -0.012277 -0.037790 -0.022203 0.014245 0.023934 0.012607 0.028185 0.041440 -0.015444 -0.022646 0.048805 0.020521 -0.080918 0.029123 -0.081415 -0.044517 0.073681 0.021481 -0.045782 -0.048161 0.003254 0.036460 -0.020011 0.065280 0.002563 -0.013150 0.045393 -0.015692 +-0.013403 0.059276 0.001032 -0.014545 0.001798 0.010646 -0.008624 -0.026081 -0.017773 0.015405 0.015492 0.014040 0.022709 0.044120 -0.010399 -0.025661 0.051172 0.017503 -0.087720 0.021120 -0.123991 -0.047505 0.087387 0.047337 -0.088646 -0.065851 0.004667 0.069448 -0.040053 0.076718 0.023178 -0.004565 0.059432 0.003138 +-0.013951 0.058265 0.004661 -0.015235 -0.000197 0.012932 -0.005151 -0.015313 -0.012719 0.014418 0.008297 0.018443 0.016678 0.051541 -0.004867 -0.028055 0.052224 0.011609 -0.091004 0.016980 -0.158439 -0.046922 0.104617 0.077207 -0.137148 -0.078657 0.005486 0.103939 -0.062683 0.082257 0.041975 0.000801 0.070429 0.027527 +-0.012210 0.028100 -0.049457 -0.000857 0.000872 -0.019030 0.038293 -0.002178 0.022197 0.008799 -0.017828 -0.005589 -0.010209 -0.026452 0.021247 -0.052723 0.074659 0.058170 -0.117953 -0.020708 -0.092993 -0.079196 0.037883 0.239302 -0.027228 -0.100648 0.058909 0.082147 0.014782 0.033906 0.058190 -0.035069 -0.021313 -0.027376 +-0.009373 0.029412 -0.026531 -0.009199 -0.003610 -0.013154 0.015847 0.006945 0.008617 0.011675 -0.022788 0.000421 -0.007697 0.005751 0.004594 -0.044864 0.063024 0.031416 -0.086743 -0.005786 -0.098907 -0.064158 0.049203 0.156108 -0.042964 -0.089257 0.038446 0.080608 -0.015415 0.071399 0.069731 -0.012170 0.005798 -0.005172 +-0.007108 0.026011 0.001392 -0.008785 -0.008676 0.005534 -0.002323 0.008151 -0.006538 0.012718 -0.016466 0.016828 0.006599 0.032792 -0.008449 -0.031395 0.045979 0.010267 -0.064263 0.010528 -0.087206 -0.035954 0.070434 0.037452 -0.075439 -0.048176 -0.000307 0.074329 -0.047401 0.093968 0.057921 0.003554 0.025206 0.029137 +-0.008616 0.022966 0.025314 -0.006647 -0.012574 0.026313 -0.017855 0.005573 -0.012147 0.015962 -0.009681 0.035391 0.015389 0.061577 -0.023327 -0.005465 0.025007 -0.014906 -0.045808 0.023055 -0.069940 -0.012007 0.101986 -0.066696 -0.085795 -0.002255 -0.051891 0.079767 -0.057766 0.078189 0.009062 0.022309 0.030125 0.051138 +-0.007559 0.013536 0.044396 -0.007128 -0.015567 0.038755 -0.032699 0.009439 -0.018290 0.012901 -0.008295 0.047013 0.018435 0.076580 -0.045586 0.016721 0.010805 -0.037831 -0.013755 0.026798 -0.057710 -0.005862 0.113804 -0.159457 -0.085242 0.031892 -0.090113 0.066240 -0.055402 0.053706 -0.034191 0.047815 0.018971 0.071248 +-0.009091 0.030428 0.013028 0.035270 -0.081955 0.036276 0.100983 -0.087388 -0.023476 0.093938 -0.074920 -0.042748 0.009283 -0.052713 0.069944 0.007487 0.044433 0.011768 -0.036858 -0.036562 0.024165 -0.141056 -0.039714 -0.075826 0.052239 0.062199 -0.048408 -0.033798 0.070343 0.124688 -0.003466 0.051026 0.026850 -0.026589 +-0.006597 0.030631 0.019664 0.021478 -0.061211 0.024677 0.092608 -0.095386 -0.041650 0.068021 -0.054620 -0.036136 0.003440 -0.021689 0.046307 0.007803 0.056620 0.018329 -0.048106 -0.037355 0.044779 -0.170111 -0.043462 -0.076020 0.064547 0.063396 -0.047873 -0.054226 0.076392 0.081326 0.000747 0.074683 0.030657 -0.067772 +-0.006641 0.024694 0.004169 0.027776 -0.027517 0.013511 0.064805 -0.082940 -0.047464 0.039102 -0.047983 -0.025260 -0.012175 0.006766 0.039254 0.004585 0.108825 0.031769 -0.009359 -0.008230 0.016421 -0.164640 -0.117072 -0.007971 0.069640 0.013253 -0.033817 -0.074767 0.071010 0.026436 0.003056 0.090554 0.005886 -0.126966 +0.003124 0.009371 0.014768 0.046412 -0.023874 0.007554 0.036147 -0.057698 -0.058069 0.021256 -0.033315 -0.028152 -0.018259 0.027895 0.014161 0.004788 0.111066 0.042408 -0.022688 -0.017329 0.008180 -0.162840 -0.121148 0.012000 0.040787 0.005472 -0.028935 -0.070284 0.051087 0.002654 0.026039 0.077896 -0.025666 -0.138535 +-0.000470 0.012891 0.009958 0.038234 -0.036776 0.012638 0.068663 -0.077589 -0.036998 0.050930 -0.057174 -0.042770 -0.006013 -0.005012 0.037131 0.011314 0.092853 0.040593 -0.014258 -0.022855 0.028109 -0.170083 -0.092356 -0.017012 0.053071 0.021873 -0.039264 -0.061002 0.057296 0.043219 0.021993 0.064114 -0.018287 -0.100373 +-0.004849 0.015938 0.022983 0.026902 -0.073622 0.023429 0.089584 -0.088285 -0.026052 0.080286 -0.059955 -0.060351 0.007903 -0.035545 0.046797 0.007563 0.038369 0.030247 -0.059021 -0.056175 0.056872 -0.166273 -0.011250 -0.093711 0.035468 0.071037 -0.053530 -0.042127 0.065314 0.098363 0.023179 0.056227 0.018585 -0.031417 +-0.022520 -0.008135 -0.028919 -0.050592 0.028229 -0.006670 -0.060855 -0.025108 0.032576 -0.018823 0.066255 0.018170 0.050211 0.001974 -0.062666 -0.057057 -0.027695 0.045343 -0.016243 0.061633 0.033008 0.024487 0.074017 -0.133643 0.041960 -0.008005 0.017760 -0.020807 0.009949 -0.031184 -0.022929 0.014085 0.078201 0.073117 +-0.008675 -0.004691 -0.013081 -0.031226 0.031157 -0.009924 -0.096969 -0.029153 0.000787 -0.038194 0.087791 0.000669 0.047391 0.025733 -0.093797 -0.072336 -0.039133 0.081739 -0.024145 0.067113 0.036711 0.015968 0.050245 -0.135056 0.022818 0.008747 0.011110 -0.044613 0.004409 -0.030744 -0.014206 -0.016314 0.026864 0.033661 +-0.008350 -0.011949 -0.026508 -0.026911 0.063836 -0.024789 -0.126394 -0.011390 -0.004841 -0.063477 0.084207 0.008974 0.040823 0.048228 -0.106669 -0.072520 0.012682 0.093989 0.007098 0.087810 0.011179 0.012357 -0.005468 -0.074058 0.051617 -0.038525 0.027206 -0.052944 -0.001957 -0.069244 -0.001150 -0.012855 0.002275 -0.012184 +0.001527 -0.031469 -0.019959 -0.032691 0.084353 -0.036934 -0.139297 0.005872 -0.012615 -0.087659 0.090429 0.021235 0.029692 0.064764 -0.125564 -0.055872 0.037072 0.084147 -0.001494 0.073344 0.027889 -0.000113 -0.020299 -0.040534 0.015971 -0.035466 0.033553 -0.071962 -0.021193 -0.078379 0.024814 -0.018801 -0.021044 -0.038763 +-0.009394 -0.028927 -0.028906 -0.024460 0.073718 -0.026898 -0.120720 -0.003723 -0.012295 -0.070463 0.075708 0.021766 0.032103 0.053181 -0.101806 -0.052394 0.030694 0.112096 0.015064 0.098439 -0.005609 0.016172 -0.039627 -0.035390 0.062895 -0.064759 0.040731 -0.023003 -0.019984 -0.078158 0.013167 -0.008582 -0.012839 -0.026034 +-0.014806 -0.020586 -0.019304 -0.037248 0.040627 -0.009079 -0.093808 -0.017684 0.001926 -0.042956 0.076953 0.012620 0.038220 0.029003 -0.084598 -0.051706 -0.017717 0.096814 -0.013796 0.075862 0.017984 0.023149 0.017010 -0.093650 0.035446 -0.023135 0.026297 -0.011769 -0.011753 -0.043351 0.000060 -0.001953 0.028135 0.023804 +0.029937 -0.018527 -0.150727 0.081029 0.034186 -0.056299 0.148788 0.006526 0.091516 -0.080441 -0.035735 0.001997 0.019030 -0.077643 0.107984 -0.056258 0.031181 -0.051535 -0.027580 0.051815 0.012036 0.081437 0.046704 -0.059784 0.100904 0.025934 0.021662 -0.016937 0.039461 -0.046583 -0.092257 -0.006300 0.025170 0.075511 +0.027077 -0.012351 -0.112038 0.035893 0.028873 -0.001260 0.084827 -0.000242 0.048683 -0.054899 -0.029997 0.032628 -0.039785 -0.108273 0.045755 -0.052769 0.008202 -0.062941 -0.003272 0.105699 0.039987 0.020061 0.023151 0.020974 0.008953 0.010896 0.031997 -0.023033 0.010634 -0.071553 -0.062459 -0.034568 0.012175 0.063440 +0.008543 0.000575 -0.036234 0.011338 0.007897 -0.004413 0.042000 0.015857 0.024113 -0.029744 -0.022255 0.053502 -0.038865 -0.097944 0.017588 -0.046984 0.033087 -0.053276 0.001529 0.130264 0.069392 -0.067173 0.042421 0.083647 -0.052001 0.027113 0.009719 0.006872 -0.038341 -0.067952 0.010640 -0.078205 -0.015510 0.044074 +-0.004854 -0.002651 -0.001199 -0.004656 -0.017881 0.006502 0.011106 0.033451 0.012608 -0.007254 -0.021085 0.015936 -0.015926 -0.005386 0.006492 -0.032176 -0.004622 -0.056644 0.008831 0.017207 0.029211 -0.031234 -0.017190 0.017370 -0.014199 -0.016837 0.012242 0.028844 0.024051 -0.033052 0.005809 -0.026851 -0.003806 0.029867 +-0.015813 -0.010424 0.049608 -0.025236 -0.047180 0.033566 -0.017722 0.037312 0.003478 0.019989 -0.017899 -0.019776 0.013111 0.105673 -0.000689 -0.029181 -0.055355 -0.059296 0.016272 -0.126448 -0.003426 0.027239 -0.080562 -0.048943 0.063203 -0.071188 0.018085 0.055066 0.103002 0.015990 -0.024552 0.027166 0.005359 -0.008937 +-0.030164 -0.027591 0.112130 -0.046006 -0.063466 0.021257 -0.061882 0.075039 -0.009550 0.046188 -0.025234 -0.036147 0.006893 0.099786 -0.046972 -0.003808 -0.042536 -0.034124 0.072153 -0.146706 -0.007001 0.011713 -0.032354 -0.013285 0.064616 -0.076540 -0.019793 0.045941 0.057997 -0.041981 -0.013777 -0.007784 -0.005500 0.063654 +-0.019028 -0.051789 0.153268 -0.072220 -0.068001 0.053218 -0.114782 0.100188 -0.032458 0.065327 -0.041993 0.000529 -0.049657 0.064999 -0.113438 0.028145 -0.026783 -0.004495 0.083060 -0.067097 0.025177 -0.054527 0.012232 0.061208 0.035559 -0.041029 -0.084866 0.011647 -0.034824 -0.100697 -0.069165 -0.000768 0.005575 0.110722 +-0.028944 -0.046124 0.115567 -0.042613 -0.067724 0.014225 -0.060491 0.103470 0.006232 0.039413 -0.031752 -0.037167 -0.006997 0.085995 -0.038657 0.010688 -0.040086 -0.042208 0.098912 -0.130371 0.037304 0.026656 -0.077987 -0.030978 0.062407 -0.072535 -0.073939 0.000909 0.019295 -0.034458 0.004467 -0.047183 0.050673 0.035176 +-0.010823 -0.035975 0.055356 -0.020108 -0.053716 0.014079 -0.008891 0.083913 0.022598 0.005824 -0.031201 -0.038542 -0.005753 0.089595 0.015986 -0.004255 -0.071317 -0.065118 0.079317 -0.106829 0.072446 0.039057 -0.148855 -0.072824 0.059724 -0.064924 -0.048338 0.013216 0.059644 0.037032 0.035514 -0.058015 0.052789 -0.068904 +0.002749 -0.025782 0.001067 -0.000955 -0.021760 -0.004910 0.022410 0.073031 0.031948 -0.017623 -0.036578 0.002064 -0.036635 -0.028107 0.022568 -0.013938 -0.026049 -0.064352 0.076825 0.032919 0.104873 -0.018636 -0.086478 -0.011360 -0.025080 -0.005483 -0.057313 -0.019218 -0.018251 -0.006270 0.058547 -0.109717 0.043965 -0.042968 +0.019557 -0.022238 -0.040271 0.015531 -0.000923 -0.011734 0.050924 0.059961 0.044488 -0.037512 -0.047698 0.041131 -0.067325 -0.125904 0.038185 -0.026749 0.003161 -0.063752 0.059945 0.149873 0.138916 -0.043073 -0.038810 0.040424 -0.060299 0.036249 -0.057345 -0.030897 -0.069820 -0.039359 0.051438 -0.140784 0.040745 -0.023033 +0.038295 -0.025011 -0.119022 0.039087 0.023303 -0.007108 0.097748 0.031451 0.066314 -0.064618 -0.049757 0.022319 -0.054682 -0.131953 0.060803 -0.041005 -0.002889 -0.065952 0.048990 0.118731 0.087371 0.038178 -0.008116 -0.019859 0.013619 0.027323 -0.017437 -0.041034 -0.017123 -0.056194 -0.033365 -0.067470 0.076076 0.024548 +0.027485 -0.031004 -0.155083 0.076967 0.030238 -0.066616 0.146021 0.018614 0.095435 -0.090351 -0.026858 -0.001195 0.004065 -0.062969 0.102157 -0.046214 0.025250 -0.047454 0.004391 0.045916 0.011457 0.054752 0.064190 -0.063966 0.092363 0.019011 0.007010 -0.017645 0.023346 -0.068486 -0.085984 -0.024607 0.046082 0.096832 +0.011663 0.006518 -0.034439 0.013241 0.007617 -0.009766 0.044050 0.033840 0.029582 -0.028393 -0.028713 0.055641 -0.043922 -0.099413 0.025141 -0.040381 0.025727 -0.058214 0.010448 0.139032 0.097498 -0.059681 0.018053 0.084934 -0.052343 0.037634 -0.003643 0.010254 -0.050500 -0.038878 0.012009 -0.092562 -0.007898 0.022313 +-0.002313 0.000470 0.000002 -0.004007 -0.016215 0.003976 0.010717 0.048028 0.016739 -0.006186 -0.024483 0.020048 -0.016876 -0.003548 0.010717 -0.026953 -0.010998 -0.055979 0.019518 0.025395 0.050851 -0.031442 -0.027433 0.021960 -0.018106 -0.009491 0.000531 0.037647 0.007463 -0.008452 0.013288 -0.048925 0.006064 0.024257 +-0.012568 -0.006592 0.049539 -0.024277 -0.046609 0.030396 -0.019942 0.053337 0.007757 0.021468 -0.022706 -0.013044 0.011272 0.109312 0.003377 -0.022628 -0.060616 -0.061807 0.030161 -0.121278 0.019402 0.026839 -0.095660 -0.042434 0.064863 -0.064156 0.007525 0.057258 0.096057 0.035786 -0.019042 0.010115 0.006232 -0.015055 +-0.023212 -0.060246 0.157807 -0.069886 -0.065658 0.063342 -0.104423 0.082001 -0.030749 0.079114 -0.044460 0.004171 -0.030429 0.042545 -0.096287 0.021890 -0.044874 -0.001170 0.067071 -0.085689 0.021431 -0.035122 0.002907 0.064224 0.017922 -0.037758 -0.083689 0.004437 -0.037603 -0.087459 -0.078257 -0.011921 0.002713 0.130484 +-0.010961 -0.010595 0.057062 -0.023094 -0.047200 0.021346 -0.011516 0.068656 0.015527 0.014810 -0.022984 -0.021018 0.006940 0.106574 0.011226 -0.023889 -0.062806 -0.069001 0.044159 -0.108003 0.050370 0.047099 -0.112640 -0.068669 0.081751 -0.068521 -0.039172 0.022972 0.052837 0.036443 0.008144 -0.028320 0.041313 -0.064570 +0.001631 -0.003250 0.002325 -0.002118 -0.016181 0.000274 0.016759 0.060910 0.024499 -0.012519 -0.030588 0.015271 -0.023100 -0.011757 0.016440 -0.028638 -0.012990 -0.064844 0.039739 0.031429 0.079984 -0.015321 -0.052897 -0.006078 -0.010170 -0.011420 -0.050449 -0.009900 -0.025391 -0.012909 0.032748 -0.085004 0.038699 -0.043162 +0.017805 0.004146 -0.038887 0.014222 0.006581 -0.009683 0.047461 0.046887 0.038038 -0.032345 -0.032763 0.056280 -0.049485 -0.108966 0.031645 -0.041431 0.020138 -0.064023 0.031810 0.142533 0.126216 -0.050995 -0.002245 0.056266 -0.050403 0.037812 -0.053115 -0.030572 -0.081976 -0.050982 0.028974 -0.127634 0.029303 -0.029574 +0.012632 -0.105919 0.042832 -0.056614 0.095894 -0.023994 -0.019214 -0.026746 -0.070577 0.202269 0.218185 -0.141116 0.012637 -0.191347 0.155533 -0.244488 0.121779 -0.013130 0.138472 -0.059114 -0.216544 -0.050477 0.066853 -0.149560 -0.157390 -0.095098 -0.011175 -0.068427 0.210754 -0.054577 0.001436 0.040342 -0.004236 -0.006205 +-0.009740 -0.029067 0.036475 -0.046866 0.118228 0.012225 -0.006405 -0.060398 -0.086556 0.108100 0.186918 -0.112111 0.050136 -0.100126 0.105268 -0.169540 0.136663 -0.073114 0.067763 -0.014521 -0.082623 -0.012863 0.033765 -0.100967 -0.086562 -0.092416 -0.077945 0.022729 0.114331 -0.008651 -0.069342 0.005683 0.051571 -0.009558 +-0.031301 0.046217 0.033243 -0.035388 0.123785 0.052248 0.021922 -0.090547 -0.125667 0.007581 0.182378 -0.055823 0.074368 -0.029074 0.053632 -0.049594 0.107463 -0.137761 0.005821 -0.001673 0.066449 0.006118 0.023528 0.013804 -0.007269 -0.081388 -0.113434 0.070675 0.027780 0.045024 -0.106739 -0.011678 0.031635 0.044212 +-0.047441 0.105188 0.031918 -0.024384 0.114098 0.082601 0.048491 -0.112125 -0.170704 -0.059301 0.164752 -0.009947 0.090739 0.006616 0.010369 0.042357 0.050623 -0.158918 -0.042210 0.014508 0.136027 -0.007670 -0.034495 0.072312 0.050835 -0.067209 -0.075955 0.023597 0.006577 0.084867 -0.082590 0.024191 -0.041988 0.068711 +-0.060936 0.133902 0.031530 -0.004574 0.064304 0.088650 0.058070 -0.124018 -0.171917 -0.102988 0.148276 0.058157 0.076201 0.044152 0.002505 0.090897 -0.015203 -0.110439 -0.090091 0.069662 0.172459 -0.039446 -0.067579 0.050794 0.093654 -0.082964 -0.026363 0.019254 -0.002660 0.016061 0.024545 0.043909 -0.092051 0.131171 +-0.074646 0.131695 0.040257 0.018490 -0.004794 0.081236 0.061589 -0.133348 -0.147784 -0.119212 0.094712 0.084045 0.033173 0.036408 0.031797 0.094617 -0.048706 0.003825 -0.075566 0.078798 0.127307 -0.065285 -0.109546 -0.039558 0.030185 -0.072564 0.052974 -0.008825 -0.088317 -0.022192 0.091494 0.087785 -0.062428 0.116086 +-0.090236 0.103626 0.066939 0.057319 -0.079163 0.060686 0.053354 -0.131920 -0.087102 -0.115977 0.028577 0.074951 -0.040556 0.035491 0.087538 0.018752 -0.081645 0.116761 0.007481 0.045592 0.001466 -0.094081 -0.126988 -0.146376 -0.033849 -0.021719 0.127080 -0.004981 -0.202755 -0.119973 0.081963 0.106640 -0.105281 -0.008628 +-0.101059 0.055177 0.092659 0.069983 -0.150193 0.026469 0.028574 -0.115258 -0.008902 -0.088879 -0.036594 0.022392 -0.109572 -0.005240 0.149355 -0.115056 -0.119773 0.122550 0.083092 0.011822 -0.092288 -0.074034 -0.034558 -0.115984 -0.044921 0.007854 0.164244 0.014803 -0.170980 -0.127570 0.019750 0.095694 0.000371 -0.041291 +-0.101563 0.024376 0.106668 0.071318 -0.187231 0.012534 0.023991 -0.099795 0.030129 -0.076617 -0.069392 -0.016558 -0.116462 -0.024845 0.158655 -0.199665 -0.116228 0.076206 0.060656 -0.024832 -0.096681 -0.049299 0.082420 -0.042295 -0.032862 0.018749 0.125343 0.039212 -0.037558 -0.047906 -0.025870 -0.049529 0.088169 -0.088273 +-0.101840 -0.013119 0.089419 0.075166 -0.182579 0.048658 0.037188 -0.067749 0.031914 -0.101814 -0.073274 -0.026481 -0.109668 -0.000858 0.110793 -0.202483 -0.060253 0.007145 0.019590 -0.069464 -0.056870 0.023689 0.132006 0.031411 0.013119 -0.007012 0.038285 0.031656 0.001551 -0.025447 -0.065416 -0.114616 0.017261 -0.086359 +-0.084252 -0.076412 0.065909 0.060881 -0.142931 0.092286 0.073191 -0.013071 0.017189 -0.159961 -0.092115 -0.031651 -0.101432 -0.005721 0.027741 -0.106024 0.050453 -0.034217 -0.067543 -0.069923 -0.011090 0.116035 0.100208 0.041462 -0.024840 -0.044467 -0.108854 -0.009354 0.073244 0.020455 -0.063103 -0.129646 -0.010099 -0.129133 +-0.053651 -0.123308 0.046845 0.020630 -0.076671 0.113112 0.110897 0.086910 0.003931 -0.182808 -0.094838 -0.023499 -0.059496 -0.018019 -0.032692 -0.004557 0.053504 -0.066417 -0.110231 -0.014603 -0.024896 0.113252 0.016412 0.021225 -0.025860 -0.075046 -0.145044 -0.030890 0.110020 0.003382 0.007818 -0.098244 -0.037896 -0.009802 +-0.023018 -0.146263 0.038114 -0.004028 -0.020148 0.111274 0.136014 0.170400 0.002465 -0.171044 -0.075670 -0.007542 -0.006424 -0.025856 -0.031416 0.083827 0.010568 -0.069980 -0.109310 0.034905 -0.065427 0.031177 -0.044852 0.019905 -0.066999 -0.093510 -0.115084 -0.095158 0.120015 0.040204 0.030019 -0.030543 -0.096811 0.096700 +0.002563 -0.127886 0.038658 -0.012071 0.033255 0.099861 0.164565 0.238246 0.001567 -0.105923 -0.057783 0.008664 0.027296 -0.030573 -0.009853 0.117020 -0.032701 -0.000283 -0.057331 0.051644 -0.084379 -0.039476 -0.031627 -0.008417 -0.048362 -0.080557 -0.044505 -0.140295 0.129312 0.090377 0.020906 0.080368 -0.088585 0.136852 +0.022743 -0.067070 0.035835 -0.011617 0.073365 0.063325 0.146213 0.262278 -0.002566 -0.007656 -0.026090 0.035984 0.031533 -0.009645 0.044980 0.084848 -0.046015 0.086576 -0.001782 0.061428 0.003570 -0.047340 0.031674 0.003656 -0.006269 -0.074342 -0.011328 -0.102136 0.042874 0.026755 -0.027391 0.130301 -0.059867 0.018942 +0.044270 0.008816 0.045389 -0.021660 0.105411 0.011416 0.109320 0.275577 -0.009832 0.108035 0.004491 0.064390 0.012428 0.006669 0.095357 -0.012238 -0.068972 0.152817 0.075822 0.056884 0.138813 -0.051472 0.080042 0.012909 0.015075 -0.053459 0.028833 0.077070 0.026517 -0.070675 -0.045298 0.143809 -0.084532 -0.083487 +0.062375 0.088371 0.056930 -0.031399 0.124205 -0.042608 0.070908 0.302850 -0.030713 0.221806 0.047883 0.094536 -0.004204 0.009990 0.148794 -0.108520 -0.096997 0.187564 0.140322 0.044017 0.262141 -0.062547 0.161527 -0.013297 0.037476 -0.028287 0.047541 0.256555 -0.040101 -0.087736 0.006879 0.108453 -0.022617 -0.092782 +0.101351 -0.059896 0.089671 -0.007831 -0.037385 0.008829 -0.001672 -0.180136 0.189267 0.006458 -0.232031 -0.060867 0.181227 0.130852 -0.046776 0.035489 0.001493 0.160017 0.051721 0.209411 0.059181 -0.083851 0.207104 -0.102355 0.039265 -0.194572 -0.024321 -0.005820 -0.022768 0.136793 -0.107813 -0.023127 -0.048810 0.065417 +0.081528 -0.068502 0.070887 -0.055525 -0.045082 0.007139 -0.005518 -0.118178 0.156592 -0.033074 -0.063860 0.040561 0.111628 0.029568 0.001045 0.060952 0.022298 0.185051 0.009521 -0.033300 0.094005 0.013600 0.105380 0.040334 -0.061870 -0.114470 0.021331 0.005113 0.002683 0.024141 -0.026811 -0.015689 0.020153 0.092373 +0.068413 -0.083706 0.052305 -0.086296 -0.055880 0.021816 0.004492 -0.068120 0.145758 -0.102567 0.092434 0.075123 0.007851 0.008520 0.005845 0.082068 0.031453 0.180413 -0.035339 -0.133303 0.088724 0.067980 0.008976 0.111553 -0.060575 -0.053669 0.037053 -0.004075 0.049025 0.024206 0.031083 0.032459 0.049141 0.038812 +0.072230 -0.065201 0.036058 -0.094849 -0.081347 0.028568 0.008797 -0.040623 0.140707 -0.126779 0.170257 0.059293 -0.052776 0.003182 -0.000677 0.081645 0.026914 0.131691 -0.055588 -0.098342 0.031136 -0.006790 0.015095 0.065597 0.078600 0.017709 0.045791 -0.034864 0.076622 0.013773 0.067413 -0.028287 0.059081 0.012658 +0.085169 -0.029802 0.042368 -0.082336 -0.097939 0.016608 0.019748 -0.012989 0.121625 -0.087612 0.209630 0.002191 -0.126582 0.050998 -0.030648 0.060010 0.000332 0.046455 -0.023080 0.055862 -0.053376 -0.135321 0.065824 0.029844 0.265716 0.072053 0.059330 -0.027794 0.089039 -0.055593 0.088900 -0.113375 0.061857 -0.009200 +0.083666 0.022002 0.040552 -0.085899 -0.099865 0.015644 0.005068 0.015547 0.156486 -0.077980 0.202180 0.014951 -0.087338 0.065851 -0.063459 -0.028938 0.043400 -0.024140 -0.005938 0.066480 -0.072232 -0.179693 -0.014319 -0.032909 0.209669 0.122452 -0.039648 0.049486 0.011409 -0.011759 -0.007885 -0.033417 -0.079718 0.113528 +0.083042 0.057031 0.040177 -0.099114 -0.076680 0.009479 -0.003028 -0.000359 0.165022 -0.119377 0.166816 0.098159 -0.004888 0.020345 -0.028135 -0.047425 0.054928 -0.014242 -0.021031 -0.044814 0.032783 -0.054961 -0.131702 -0.039012 -0.016037 0.136114 -0.052413 0.121523 -0.038968 0.020495 -0.008329 -0.024960 -0.081805 0.022575 +0.083252 0.078107 0.056261 -0.095219 -0.059524 0.003298 -0.006615 -0.018114 0.165684 -0.099090 0.091701 0.078091 0.033441 -0.005544 -0.024775 -0.060616 0.054683 -0.025820 0.034770 -0.091304 0.061625 0.040350 -0.178139 0.001904 -0.150915 0.076588 -0.034127 0.096746 -0.053605 0.008977 -0.035470 0.025028 -0.047913 -0.054413 +0.104239 0.086773 0.093349 -0.055643 -0.033261 -0.013246 -0.005449 -0.083837 0.158899 -0.044047 -0.029467 -0.026012 0.082055 -0.049138 0.021797 0.008367 0.031893 -0.087107 0.189897 -0.063379 0.017963 0.128801 -0.143253 0.036421 -0.143812 -0.013284 0.024955 0.056737 -0.069906 0.014591 -0.040838 0.026539 -0.056752 -0.047314 +0.135218 0.100220 0.124814 0.000991 -0.008239 -0.012542 -0.007248 -0.156811 0.151583 -0.017420 -0.147099 -0.155713 0.101870 -0.021523 0.028922 0.110680 0.026595 -0.179864 0.346267 0.114265 -0.070904 0.161901 -0.031710 0.014850 -0.024451 -0.120325 0.049384 -0.013378 -0.123593 0.072623 -0.129874 0.065687 -0.065814 -0.083173 +0.059343 -0.012441 -0.000702 -0.081890 -0.008092 -0.077276 -0.028878 0.037295 -0.029467 -0.011411 -0.007775 0.009769 -0.027726 0.005769 0.053610 0.007873 -0.041172 -0.106420 -0.027219 0.033050 -0.102197 -0.048661 -0.066981 -0.013190 0.109772 0.030705 0.044138 0.003928 0.036674 0.004169 0.127857 0.033634 0.266703 0.060249 +0.041616 -0.014486 -0.010645 -0.063688 0.002398 0.007759 -0.030515 0.045130 -0.060320 -0.012907 -0.003144 -0.002022 0.026184 0.062737 0.093489 -0.012212 -0.012731 -0.061123 0.017645 0.059738 -0.070951 0.028276 -0.008200 -0.013282 0.059036 0.130001 0.007378 -0.026986 0.024631 0.046745 0.129132 0.029581 0.168902 0.047167 +0.022209 -0.018785 -0.021705 -0.047282 0.007684 0.096519 -0.037379 0.046370 -0.086593 -0.016226 0.004486 -0.016435 0.077777 0.123311 0.134030 -0.031013 0.005692 -0.004530 0.059412 0.082817 -0.053324 0.097630 0.047516 -0.011465 0.006049 0.229178 -0.021088 -0.047042 0.006557 0.091702 0.122335 0.003955 0.066979 0.045888 +0.003588 -0.021043 -0.030263 -0.030409 0.008371 0.181691 -0.044518 0.046051 -0.107937 -0.022794 0.009934 -0.032445 0.119848 0.179589 0.164240 -0.043652 0.020389 0.047092 0.094236 0.116267 -0.035936 0.155206 0.107422 -0.005142 -0.059549 0.322080 -0.057313 -0.069079 -0.015886 0.152295 0.127300 -0.018228 -0.013766 0.025152 +0.010241 -0.012175 -0.071132 -0.010497 0.020024 0.062831 -0.040007 0.039616 -0.072378 0.035685 -0.031690 0.021640 0.001871 0.085038 0.053829 -0.017137 0.002122 0.049195 0.087728 -0.000013 -0.026558 0.022211 -0.058457 0.166112 0.018559 -0.012076 -0.001099 0.016641 -0.019024 0.008824 0.091310 -0.040634 0.020518 0.051538 +0.005987 -0.016904 -0.065972 -0.010944 0.009374 0.084910 -0.052019 0.031735 -0.073636 0.036433 -0.016286 0.003295 0.010276 0.085334 0.068926 -0.031102 -0.004326 0.037220 0.078248 0.028015 -0.030765 0.043535 -0.024633 0.098897 0.024402 0.055196 -0.037672 -0.019356 0.027747 -0.077052 0.014090 -0.032531 0.010393 0.037211 +0.000485 -0.009792 -0.056592 -0.011803 0.001583 0.104774 -0.061755 0.023501 -0.079032 0.039804 -0.011369 -0.011280 0.024653 0.096826 0.073638 -0.046150 -0.008896 0.030450 0.076726 0.038805 -0.033667 0.064954 0.000900 0.056790 0.020232 0.116360 -0.065525 -0.045163 0.037248 -0.094933 -0.027418 -0.028737 -0.007585 -0.011135 +0.006045 -0.001260 -0.070490 -0.009554 0.009933 0.083302 -0.054060 0.013771 -0.080073 0.047390 -0.001836 -0.009542 0.020186 0.082461 0.052009 -0.037023 0.003870 0.028911 0.066158 0.019377 -0.061487 0.042407 -0.017089 0.104383 0.041935 0.073886 -0.019325 -0.089932 0.029586 -0.070428 0.024956 -0.057337 0.001123 -0.019238 +0.011242 -0.005679 -0.074412 -0.010055 0.015755 0.060586 -0.047635 0.003394 -0.091558 0.044380 -0.005161 -0.022962 0.015788 0.070458 0.028218 -0.036979 0.017605 0.028455 0.076360 -0.013275 -0.085415 0.018537 -0.048259 0.163691 0.057659 0.026190 0.028125 -0.133961 0.018631 -0.042571 0.090055 -0.092227 0.027934 -0.057308 +0.036225 -0.071534 0.000459 -0.046818 0.009337 -0.083866 -0.046178 -0.015273 -0.060642 -0.002822 -0.089257 0.012351 -0.039814 -0.035023 -0.019465 0.040371 -0.061960 0.039386 -0.094567 -0.024135 0.027052 0.059938 0.091591 0.001979 -0.040503 -0.107158 0.083539 -0.101314 0.013352 -0.110938 0.043372 0.129228 0.064399 -0.008668 +0.044367 -0.059525 0.015036 -0.064121 -0.018004 -0.082253 -0.045795 -0.026919 -0.070584 -0.018065 -0.070391 -0.010245 -0.029751 -0.046835 -0.028796 0.042482 -0.143375 0.002483 -0.181009 -0.070951 0.071994 -0.003359 0.199769 -0.097579 -0.087064 0.021258 0.050995 -0.107355 -0.017802 0.053633 0.017380 0.028065 0.073668 -0.008871 +0.043601 -0.049111 0.016832 -0.062722 -0.017690 -0.081388 -0.042392 -0.021628 -0.089830 -0.013515 -0.066851 -0.008770 -0.034597 -0.040346 -0.030496 0.037791 -0.131468 -0.012470 -0.162906 -0.049426 0.049875 -0.008340 0.154951 -0.079562 -0.052199 0.026980 0.057094 -0.092234 -0.013567 0.075758 0.017100 -0.027829 0.047598 -0.032337 +0.048275 -0.038263 0.001498 -0.051199 0.023312 -0.088975 -0.041336 0.005000 -0.095004 -0.003208 -0.060932 0.022466 -0.042236 0.005051 -0.023424 0.039786 -0.019435 -0.003108 -0.056234 0.026974 -0.047324 0.009209 -0.052575 0.062674 0.041095 -0.089544 0.071103 -0.050484 0.024541 0.021517 -0.001177 -0.034659 0.017784 -0.145986 +0.041785 -0.047523 -0.029972 -0.046078 0.035909 -0.097136 -0.056514 0.005071 -0.089750 -0.024247 -0.085315 0.019769 -0.065103 -0.012747 -0.033756 0.024734 -0.014857 0.011973 -0.055355 0.034763 -0.048361 0.024915 -0.036487 0.063963 0.019368 -0.114169 0.095381 -0.092460 0.033839 -0.059458 -0.018405 0.004700 0.059549 -0.091842 +0.044692 -0.060996 -0.030403 -0.035242 0.035271 -0.092818 -0.058749 -0.002830 -0.079239 -0.025316 -0.092148 0.019888 -0.062782 -0.020325 -0.029536 0.030127 -0.024623 0.028330 -0.061266 0.010928 -0.030107 0.035277 -0.006948 0.059755 -0.013675 -0.115893 0.086825 -0.110997 0.020747 -0.090179 -0.023240 0.046759 0.032906 -0.074567 +0.053635 0.024179 0.003976 -0.054942 0.021919 -0.085630 -0.040517 0.019222 -0.098845 -0.002500 -0.058854 0.005767 -0.030366 0.004222 -0.039686 -0.024247 -0.003591 -0.045881 -0.051159 0.028487 -0.059601 0.002317 -0.116937 0.048023 0.058109 -0.042187 -0.022689 0.101089 -0.085146 -0.047060 0.001347 -0.071920 -0.053922 -0.029344 +0.050654 0.043811 0.016440 -0.071448 -0.018825 -0.083915 -0.050552 0.001389 -0.099731 -0.011661 -0.055724 -0.038660 -0.011181 -0.044498 -0.046592 -0.061600 -0.110975 -0.092581 -0.146722 -0.054465 0.022640 -0.027495 0.052884 -0.124647 -0.044100 0.090185 -0.056166 0.049620 -0.094099 0.024798 -0.007076 -0.078109 -0.056294 0.024809 +0.056727 0.056174 0.016328 -0.067723 -0.018273 -0.089164 -0.053246 -0.004474 -0.088775 -0.016960 -0.055755 -0.049357 -0.003393 -0.057029 -0.057944 -0.078602 -0.116320 -0.116419 -0.135346 -0.065112 0.031714 -0.026089 0.053926 -0.146501 -0.088218 0.112238 -0.073090 0.019614 -0.073331 -0.010391 -0.016316 -0.081091 -0.101346 0.033206 +0.053863 0.070189 0.002551 -0.055065 0.015300 -0.088786 -0.052051 0.006288 -0.079362 0.003819 -0.069097 -0.028654 -0.012574 -0.043552 -0.060415 -0.082841 -0.030205 -0.090290 -0.056654 -0.010515 -0.026259 0.018692 -0.060457 -0.049003 -0.020092 0.013016 -0.079443 0.055920 -0.044095 -0.132379 -0.069462 -0.028557 -0.081805 -0.014368 +0.054487 0.052156 -0.025662 -0.043672 0.038234 -0.097652 -0.061550 0.009141 -0.091552 -0.018906 -0.073348 -0.023368 -0.035182 -0.019426 -0.067143 -0.077651 0.000840 -0.064016 -0.050794 0.012015 -0.070276 -0.009742 -0.116913 0.009872 0.004210 -0.020941 -0.054064 0.033929 -0.041700 -0.124288 -0.068147 -0.077671 -0.106441 -0.047888 +0.053071 0.036339 -0.025735 -0.047996 0.041005 -0.100444 -0.057829 0.013845 -0.101921 -0.018419 -0.077645 -0.013513 -0.041059 -0.010444 -0.059771 -0.061024 0.005044 -0.056949 -0.052826 0.026317 -0.078409 -0.007588 -0.129995 0.033848 0.029133 -0.047594 -0.035022 0.047235 -0.058100 -0.091996 -0.054179 -0.066075 -0.078103 -0.046683 +-0.031460 -0.013987 -0.149818 0.080350 0.020681 -0.057818 0.011712 -0.001008 0.061313 0.017201 0.005085 0.074818 -0.047190 0.015967 -0.029569 0.136405 0.070269 -0.039349 0.090760 -0.058563 0.065843 0.078259 -0.020924 -0.163454 -0.040312 -0.104189 0.069961 0.035861 0.064290 0.042196 0.239137 -0.024848 0.084332 -0.100068 +-0.002327 -0.017521 -0.139425 0.070865 0.048305 0.079434 -0.035213 -0.026484 0.062642 0.070813 -0.027307 0.065759 -0.050332 -0.068535 -0.050488 0.065260 -0.034808 0.001184 0.022065 -0.018875 0.057119 -0.010911 -0.054658 -0.020400 -0.028406 -0.025774 0.004056 0.020749 -0.040770 0.067283 0.024925 0.111530 0.079035 -0.073362 +-0.000584 -0.011118 -0.122177 0.051211 0.037283 0.151535 -0.057755 -0.043066 0.075106 0.105702 -0.045195 0.033969 -0.051730 -0.086152 -0.035502 0.008973 -0.086193 0.022393 -0.032203 -0.010625 0.037487 0.012267 -0.063710 0.003409 0.012534 0.018225 -0.069447 -0.045926 -0.057301 0.086177 -0.125110 0.108154 0.062487 -0.033929 +0.000480 -0.003086 -0.124476 0.057127 0.033887 0.149034 -0.061112 -0.048529 0.070950 0.110661 -0.040134 0.025767 -0.039288 -0.067746 -0.045629 0.005133 -0.081370 0.017086 -0.037104 -0.021039 0.008034 -0.013678 -0.051069 0.002511 0.032016 0.008503 -0.097044 -0.037501 -0.056407 0.051508 -0.144414 0.076356 0.059066 -0.021945 +-0.000559 0.003997 -0.124027 0.053465 0.037579 0.149996 -0.049979 -0.055144 0.070483 0.113771 -0.040946 0.023671 -0.042610 -0.080241 -0.054001 0.002411 -0.078521 0.019903 -0.057589 -0.015049 -0.008310 -0.005005 -0.034476 0.006235 0.033286 0.024151 -0.120019 0.018377 -0.072419 0.051602 -0.134832 0.064370 0.037586 0.038711 +-0.000565 0.012494 -0.144971 0.071663 0.048838 0.081814 -0.029624 -0.048941 0.058314 0.080172 -0.010279 0.030880 -0.020393 -0.055948 -0.087529 0.040786 -0.005034 0.001085 -0.046251 -0.044062 -0.050526 -0.012975 0.003798 -0.023311 0.028814 -0.002381 -0.096432 0.041930 -0.029305 0.030971 -0.027834 0.007430 -0.084175 0.004963 +-0.024116 0.030827 -0.152081 0.079655 0.025107 -0.047327 0.011967 -0.026416 0.054183 0.040100 0.018945 0.030843 -0.004122 0.017941 -0.093164 0.086158 0.096071 -0.024027 0.018254 -0.066233 -0.099375 0.036822 0.073623 -0.184943 0.021773 -0.066513 -0.059088 0.109738 0.066223 -0.037492 0.178023 -0.104653 -0.133898 -0.056362 +-0.091307 0.021185 0.005452 0.093079 0.004462 -0.137494 0.027255 0.016193 0.035027 0.043270 0.017597 -0.052594 0.050678 -0.017621 -0.100552 0.045091 0.137042 -0.020604 0.004831 0.003501 -0.076423 -0.058265 0.083175 0.046838 -0.041851 0.066991 0.022783 0.025688 0.023265 -0.031676 0.073439 -0.072413 -0.208999 0.025261 +-0.117606 0.016526 0.081068 0.093572 -0.000820 -0.179734 0.034199 0.055206 0.036610 0.045268 0.036467 -0.091173 0.086052 0.019906 -0.077335 0.021613 0.112294 -0.024854 0.001051 -0.024213 0.006556 -0.070798 0.044003 0.109037 -0.026106 0.108415 0.052460 -0.013149 -0.012349 -0.024464 -0.073565 -0.035321 -0.006109 0.085182 +-0.119255 0.013200 0.094025 0.092040 -0.002146 -0.181868 0.034095 0.066850 0.038987 0.042822 0.035705 -0.088591 0.084897 0.040278 -0.058173 0.015843 0.087899 -0.013908 0.006996 -0.042503 0.059924 -0.066579 0.004384 0.103210 -0.024232 0.105557 0.087612 -0.040335 0.005558 -0.016772 -0.091889 -0.007373 0.029448 0.052985 +-0.116290 0.010377 0.087191 0.091107 -0.002629 -0.185149 0.030103 0.072827 0.039401 0.035673 0.032489 -0.068554 0.067030 0.041143 -0.052476 0.026493 0.086061 -0.007730 0.013083 -0.014025 0.109392 -0.074283 -0.017676 0.099392 -0.055753 0.103876 0.090079 -0.044614 -0.019969 0.012981 -0.056802 -0.005694 0.062207 0.013923 +-0.088956 0.002770 0.014073 0.083735 0.000259 -0.146666 0.024624 0.057239 0.040658 0.024938 0.022137 -0.006530 0.016834 0.033547 -0.054639 0.066908 0.066243 -0.006261 0.048043 0.020086 0.160704 -0.046164 -0.050282 0.007988 -0.125443 0.055461 0.087275 -0.006875 -0.007782 0.054046 0.083770 0.004746 0.021062 -0.066056 +-0.035780 -0.013571 -0.129727 0.075906 0.019981 -0.033424 -0.002959 -0.000563 0.050482 0.021935 0.002944 0.075011 -0.045732 0.001801 -0.047564 0.120066 0.054278 -0.033545 0.090304 -0.063131 0.074862 0.062176 -0.020091 -0.117017 -0.059383 -0.080761 0.087965 0.081966 0.076590 0.065730 0.239584 -0.057628 0.009591 -0.117312 +-0.008820 -0.007627 -0.158056 0.055203 0.017956 0.077099 -0.075129 -0.007191 0.032083 0.085832 -0.010026 0.034463 -0.026295 -0.045494 -0.041653 0.027555 -0.049879 0.009335 0.083404 -0.078297 0.040478 -0.117472 0.024219 0.074484 -0.004484 -0.034318 -0.038895 -0.061118 -0.048430 -0.005360 -0.051358 0.010804 0.052828 0.036864 +-0.007737 -0.003256 -0.151791 0.055268 0.019966 0.087559 -0.078741 -0.018450 0.030698 0.091713 -0.015450 0.020864 -0.015028 -0.032981 -0.052983 0.014209 -0.059863 0.016094 0.069445 -0.092531 0.010370 -0.126361 0.028344 0.071716 0.009341 -0.033236 -0.067826 -0.044003 -0.045043 -0.018573 -0.090383 0.001302 0.035380 0.033187 +-0.007412 0.002548 -0.158757 0.061181 0.018661 0.073976 -0.071983 -0.025639 0.028395 0.091940 0.000206 0.022257 -0.009808 -0.034953 -0.058116 0.017318 -0.038657 0.005313 0.049993 -0.087077 -0.010529 -0.130728 0.062698 0.069370 0.019104 -0.031545 -0.097670 0.007100 -0.077453 -0.016369 -0.071504 -0.037358 0.022404 0.093558 +-0.032512 0.028427 -0.133433 0.078989 0.025468 -0.028533 0.004691 -0.021006 0.046417 0.047024 0.018770 0.037960 0.000477 0.007323 -0.092459 0.082200 0.086554 -0.011680 0.010444 -0.068859 -0.057549 0.014886 0.067619 -0.129370 0.001837 -0.039609 -0.014336 0.136787 0.056466 0.005406 0.166161 -0.086177 -0.153931 -0.065626 +-0.106396 0.012082 0.093804 0.073458 0.006325 -0.122138 0.053986 0.020569 0.052258 0.097040 -0.030117 -0.063935 0.029007 -0.048173 -0.073480 0.006487 0.079422 -0.018318 -0.098277 0.085998 -0.045678 0.124596 -0.056062 -0.009915 0.115255 0.082543 0.043762 0.055274 0.033651 0.018777 -0.097186 0.025652 0.107246 0.082007 +-0.108266 0.013196 0.107352 0.073267 0.005501 -0.118182 0.053900 0.026988 0.059629 0.098678 -0.033463 -0.059244 0.030748 -0.021010 -0.048874 -0.003174 0.046149 -0.004115 -0.096188 0.073125 -0.013743 0.137679 -0.084938 -0.025411 0.119975 0.081742 0.078842 0.041866 0.054237 0.044745 -0.113358 0.068330 0.125398 0.071168 +-0.105677 0.013856 0.100276 0.068541 0.007179 -0.120756 0.052194 0.033207 0.056095 0.092773 -0.030246 -0.037497 0.018805 -0.020756 -0.042271 0.009948 0.041864 0.000654 -0.101674 0.096004 0.022386 0.126403 -0.085055 -0.027232 0.087129 0.079991 0.091934 0.037413 0.035609 0.080869 -0.077055 0.068696 0.147811 0.051447 +0.200300 0.206381 0.037666 -0.062044 0.105603 0.039287 0.129443 -0.026108 -0.006145 0.008167 -0.180506 -0.005344 -0.154896 0.029728 -0.078524 -0.093022 0.282292 0.094500 -0.012038 -0.075772 -0.007254 0.075631 -0.021364 -0.032209 0.106197 -0.033075 -0.057528 -0.036911 -0.079279 -0.070271 0.134924 0.050103 0.176129 0.174669 +0.174909 0.262626 0.001650 -0.050760 0.056767 0.046800 0.116679 0.016697 0.036953 -0.001587 -0.129708 -0.018479 -0.077744 0.017170 -0.006678 -0.084015 0.149428 0.003802 0.023948 -0.034466 -0.037293 0.015672 -0.028566 0.018799 0.029636 -0.058638 0.022368 -0.067504 -0.036674 -0.038199 0.101980 -0.006688 0.114199 0.092127 +0.145419 0.271569 -0.034873 -0.044417 -0.005370 0.054598 0.106804 0.058877 0.047845 -0.006222 -0.038196 -0.028624 0.051275 0.021585 -0.019610 -0.032440 -0.042228 -0.040054 0.010931 -0.080129 -0.032979 -0.037624 0.046956 0.005674 -0.043045 -0.070160 0.084897 -0.059861 0.011866 -0.020727 0.016502 -0.105536 0.054309 0.077813 +0.121784 0.205203 -0.040615 -0.025224 -0.078037 0.055910 0.087096 0.101567 0.032281 0.047114 0.067450 -0.029754 0.115044 0.004978 -0.002689 0.033261 -0.183060 -0.118272 -0.032756 -0.083449 -0.036678 -0.109978 0.089839 -0.018425 -0.047398 -0.071371 0.132785 -0.079312 0.053450 -0.016013 -0.087251 -0.136778 0.010646 0.048749 +0.133914 0.102310 -0.020287 0.004986 -0.141709 0.003779 0.009098 0.120102 -0.047440 0.060693 0.150862 0.032856 0.123342 0.036340 0.053349 0.081323 -0.106337 -0.089787 -0.055918 0.076288 0.022085 -0.032767 0.044374 -0.049343 -0.054487 -0.077724 0.094554 -0.099233 0.070220 0.032782 -0.134028 -0.176615 0.058256 -0.110604 +0.136156 0.012237 0.028350 0.055205 -0.144687 -0.046150 -0.025253 0.036940 -0.082417 0.046931 0.140621 0.125137 0.070684 0.035326 0.117788 0.115735 0.014819 -0.011779 -0.095812 0.103471 0.027698 0.015841 -0.045732 -0.086235 -0.100680 -0.049399 -0.026388 -0.094643 0.100905 0.044076 -0.187605 -0.094315 0.003539 -0.202719 +0.096825 -0.052123 0.057933 0.114355 -0.098038 -0.121617 -0.055895 -0.022068 -0.059251 -0.000028 0.053843 0.302609 -0.066475 0.059366 0.163306 0.029775 0.086102 0.037617 -0.004275 0.136346 -0.030689 0.070476 -0.032601 -0.031896 -0.043260 -0.110607 -0.181097 0.120911 0.098080 0.036017 -0.155625 -0.014093 -0.023635 -0.063091 +0.053280 -0.060018 0.037506 0.137972 0.011258 -0.178318 -0.023102 -0.051378 -0.072595 -0.086067 0.000313 0.285580 -0.051029 -0.025127 0.085024 -0.036423 0.014116 0.013590 0.100603 0.056762 -0.064465 -0.011526 0.015876 -0.005954 0.118192 -0.107172 -0.166406 0.124543 0.111421 0.028985 -0.074010 0.076955 -0.032659 0.141702 +0.045574 -0.013216 0.032142 0.093571 0.050167 -0.171049 0.052834 -0.067576 -0.122586 -0.148784 -0.054492 0.096339 -0.018259 -0.062274 -0.040250 -0.143567 -0.086938 0.063209 0.139416 -0.029860 -0.021312 -0.063067 0.038636 0.074870 0.004951 -0.015358 -0.118334 -0.004443 0.136427 0.102426 0.070536 0.141456 -0.079749 0.125498 +0.059666 0.033773 0.011922 0.066781 0.019188 -0.136499 0.012310 -0.053448 -0.112842 -0.162605 -0.033516 -0.083470 0.040366 -0.073418 -0.073634 -0.134124 -0.183406 0.070520 0.140600 -0.024180 0.078113 -0.014717 -0.030918 0.055699 -0.059861 -0.024527 -0.045187 -0.018538 0.147098 0.098916 0.092276 0.126182 -0.051521 0.096536 +0.102008 0.062652 0.002559 0.033904 -0.034709 -0.028794 -0.029963 -0.001155 -0.088109 -0.139233 -0.017772 -0.206193 0.050147 -0.078321 -0.066908 -0.098039 -0.008521 0.078240 0.003116 -0.115075 0.170119 0.063555 0.042319 0.085075 -0.085497 -0.000332 -0.058397 -0.013971 0.095261 -0.001791 0.088311 0.169500 -0.076125 -0.011915 +0.130156 0.030796 -0.028981 0.037504 -0.110897 0.045667 -0.073986 0.097997 -0.071465 -0.114585 0.024864 -0.164295 0.078517 -0.068363 -0.064291 -0.015782 0.086141 -0.030675 -0.057981 -0.071696 0.160759 0.037941 0.106627 -0.011196 0.036540 -0.017581 -0.051521 0.093436 -0.051622 -0.102102 0.046722 0.156601 0.065388 -0.029184 +0.168973 -0.048346 -0.054128 0.028599 -0.159253 0.088824 -0.040072 0.101158 -0.015747 -0.066487 0.069651 -0.136555 0.131030 -0.095612 0.005098 0.051930 0.141634 -0.059902 -0.077737 -0.001406 0.060549 0.016779 0.073755 -0.079590 0.025474 0.002819 -0.008349 0.092361 -0.102785 -0.130624 0.011078 0.134838 0.105198 -0.029608 +0.196868 -0.140552 -0.038053 0.030363 -0.136280 0.109791 0.082851 0.031947 -0.010593 0.024018 0.071666 -0.120218 0.158965 -0.056053 -0.005157 0.089872 0.082277 -0.037538 -0.032307 -0.004000 -0.077070 -0.047653 -0.036306 -0.059558 -0.007532 0.012245 0.064610 0.069522 -0.100870 -0.125037 -0.000483 0.105122 0.004543 -0.031563 +0.220722 -0.189143 -0.019812 0.050177 -0.022114 0.078768 0.141034 -0.059095 -0.020072 0.062754 0.030385 -0.038935 0.084110 -0.009296 0.007037 0.034972 0.032848 -0.022511 0.007807 0.025446 -0.154512 -0.055275 -0.065423 0.057532 0.001583 -0.000069 0.148740 0.038380 -0.084950 -0.055361 -0.005477 0.024714 -0.104505 -0.036786 +0.226640 -0.179498 0.049226 0.043510 0.105148 -0.008538 0.147024 -0.091442 -0.062929 0.109845 0.005074 0.058119 0.017767 0.089994 -0.044924 -0.062390 -0.007394 0.037390 0.010439 -0.015520 -0.064619 0.042369 -0.058993 0.055481 -0.000104 0.024385 0.166181 0.000132 -0.087429 0.021365 -0.001406 -0.118401 -0.061089 0.025621 +0.236683 -0.154942 0.112382 0.048015 0.188712 -0.046365 0.145337 -0.144361 -0.102570 0.165949 -0.050088 0.165649 -0.012631 0.142407 -0.082270 -0.087023 0.021488 0.106059 -0.048916 -0.157005 0.118604 0.107512 -0.049746 -0.120962 0.092471 0.115214 0.092679 -0.156215 -0.070360 0.083441 -0.097703 -0.215489 -0.072382 0.027100 +0.098748 -0.110249 0.040632 0.106294 0.034691 0.030852 -0.096378 -0.114844 0.076385 0.054995 -0.090801 -0.147677 0.060390 0.097064 -0.014503 -0.075035 -0.111119 -0.058225 -0.072715 0.091030 0.140176 -0.115472 -0.124135 0.002804 -0.034399 0.069021 -0.071552 0.126436 0.208688 -0.064774 0.167890 0.023198 0.065838 -0.115322 +0.045293 -0.089374 0.018089 0.085862 0.012661 -0.008385 -0.067383 -0.079743 -0.026056 0.045936 -0.103235 -0.087935 0.099554 0.036145 -0.034997 -0.024818 -0.088738 0.005542 -0.088960 0.117580 0.051157 -0.047818 -0.031283 0.050240 -0.019133 0.069263 -0.073513 0.173658 0.080920 -0.038285 0.140359 -0.084856 0.069031 -0.038620 +-0.004362 -0.070045 -0.005441 0.028721 -0.035581 0.008714 -0.023807 -0.055672 -0.056772 0.019390 -0.051096 -0.030542 0.150701 -0.048870 -0.050577 0.013535 -0.059832 0.055515 -0.059955 0.062922 0.041041 0.043294 -0.006914 0.139734 -0.004182 0.051132 -0.044750 0.176739 0.008876 -0.029887 0.077004 -0.117767 0.056034 -0.030393 +-0.046908 -0.055997 -0.030801 -0.012931 -0.095239 0.007226 0.007382 -0.049966 -0.065128 -0.036849 -0.026556 0.024951 0.158839 -0.114967 -0.061207 0.047650 -0.045883 0.050290 0.011314 0.022440 -0.012041 0.017932 0.042840 0.074523 0.063977 0.076758 -0.024231 0.082162 -0.002699 -0.024259 0.019286 -0.035404 0.016136 -0.056954 +-0.084798 -0.042180 -0.051749 -0.022690 -0.163003 -0.027359 0.032238 -0.030906 -0.064720 -0.018269 -0.007672 0.068355 0.116840 -0.135774 -0.081552 0.048596 -0.080784 0.017814 0.146383 0.045115 -0.164446 -0.039490 0.025936 0.042829 0.216153 0.010996 0.006383 0.024871 0.002893 -0.034911 -0.009591 0.024318 -0.029688 -0.025116 +-0.106381 0.037472 -0.062633 -0.029746 -0.165792 -0.010239 0.012509 -0.033520 0.075020 0.048760 0.008410 0.182188 0.245250 0.004609 -0.187742 -0.114282 -0.018821 0.047174 0.096399 0.074864 -0.071890 0.002693 0.007320 0.000897 0.070591 0.012206 -0.053249 -0.072801 0.098886 -0.111274 0.027638 0.030143 0.068139 0.024639 +-0.081730 0.078091 -0.032084 -0.016937 -0.094799 -0.034978 -0.035412 -0.016776 0.073337 0.010988 -0.014905 0.221079 0.199761 -0.067149 -0.096254 -0.041963 0.032786 0.009098 0.047313 -0.051221 -0.021669 0.051603 0.007726 -0.033954 -0.118397 0.100014 -0.080501 -0.162248 0.076284 -0.144372 0.005300 0.030886 0.089261 -0.018362 +-0.032591 0.093392 0.005011 0.033984 -0.033653 -0.056018 -0.045205 -0.034382 0.054241 0.038895 -0.028006 0.147818 0.118347 -0.066122 -0.022476 0.025348 -0.024972 -0.006291 0.042302 -0.124172 0.007567 0.044630 0.010710 0.049093 -0.093271 0.092758 -0.055927 -0.139825 0.084283 -0.141690 0.001749 -0.018718 0.034520 -0.013745 +0.025131 0.111171 0.052338 0.093959 0.036120 -0.076090 -0.055111 -0.055018 0.028419 0.039071 -0.021446 0.070711 -0.045120 0.015776 0.119003 0.157191 -0.056194 -0.037864 -0.019407 -0.101495 -0.025226 -0.020203 0.081456 0.053677 0.018315 0.139230 -0.090725 -0.024605 0.076010 -0.142354 0.011727 -0.028982 0.009696 0.010824 +0.086343 0.128544 0.089920 0.151910 0.118411 -0.058267 -0.085331 -0.086356 0.086411 0.023158 -0.022226 -0.036163 -0.177136 0.077261 0.277540 0.318083 -0.140120 -0.082366 -0.088998 -0.072522 -0.066982 0.005449 0.106803 0.086831 0.077782 0.142782 -0.071544 0.132130 0.111582 -0.160749 0.004996 0.042392 -0.012883 0.064412 +-0.061540 0.002620 -0.073689 0.003550 -0.071570 -0.007284 0.013967 -0.001633 -0.094799 -0.008264 -0.030895 0.022099 0.055730 0.034688 -0.025540 -0.038442 -0.002687 -0.049120 0.053353 0.026489 0.022921 0.048577 -0.071688 0.080905 0.064980 0.013782 -0.014355 -0.070959 -0.107091 0.226291 0.009386 0.025020 -0.005604 -0.092437 +-0.083267 -0.013381 -0.104616 -0.054605 -0.053350 -0.040950 0.035114 0.002258 -0.090956 -0.033881 -0.040826 0.018097 0.028479 0.074550 0.001071 0.021358 0.041581 -0.040821 0.048402 0.008007 0.052241 0.038045 -0.031619 0.029420 0.021501 0.017835 -0.011740 -0.040020 -0.113478 0.098608 -0.030604 0.032750 0.004352 -0.062761 +-0.112570 -0.029556 -0.131694 -0.107775 -0.048902 -0.072433 0.059086 0.003800 -0.085443 -0.052929 -0.057306 0.008400 0.005223 0.134826 0.035720 0.064559 0.065656 -0.055342 0.042012 0.015993 0.045317 0.026565 0.036061 -0.016830 -0.006164 0.013784 -0.005233 0.023644 -0.106323 0.001697 -0.058936 0.048287 0.020797 -0.018860 +-0.138208 -0.042880 -0.154716 -0.161145 -0.036682 -0.087614 0.078138 -0.004009 -0.085960 -0.078308 -0.075158 -0.013667 -0.009818 0.193081 0.069946 0.112698 0.091929 -0.063611 0.052582 0.024224 0.037179 0.031363 0.096072 -0.069509 -0.079675 0.025210 -0.001654 0.058541 -0.103434 -0.086341 -0.084715 0.091389 0.038254 0.015236 +-0.071614 -0.025433 -0.079555 -0.044231 -0.056001 0.003353 0.020448 0.039313 -0.034449 0.021201 -0.079939 0.028644 -0.024787 0.188439 0.060152 0.035784 0.042994 0.023862 -0.002200 -0.094819 -0.024939 0.021039 0.009160 -0.055080 -0.021112 -0.045149 -0.025621 0.216443 -0.029294 0.072591 -0.079226 -0.017757 0.067523 0.075291 +-0.072195 -0.021031 -0.121170 -0.072247 -0.036905 -0.070614 0.012662 0.040015 -0.043028 0.012293 -0.078126 -0.030670 -0.033902 0.118408 0.040616 0.048834 0.050552 0.038276 0.008483 -0.058060 -0.034078 -0.041976 -0.009228 -0.045368 -0.013261 -0.068158 0.022561 0.132560 0.004265 -0.025965 -0.059467 -0.019725 0.063179 0.097074 +-0.083844 -0.027702 -0.148124 -0.094158 -0.023167 -0.103209 0.009646 0.023882 -0.042179 0.000862 -0.043166 -0.071528 -0.042289 0.065372 0.035839 0.017436 0.064917 0.010734 0.049860 -0.058510 -0.037633 -0.085134 -0.033890 -0.026309 -0.004119 -0.070958 0.080402 -0.027002 0.048232 -0.088265 -0.053340 0.014549 0.001863 0.085118 +-0.077701 -0.032876 -0.116524 -0.069424 -0.039070 -0.071006 0.000686 0.000032 -0.060440 -0.009999 -0.033027 -0.093775 -0.041865 0.049000 0.019074 0.068912 0.042780 0.002562 0.084029 -0.024194 0.004517 -0.150073 -0.009540 -0.075583 0.001970 -0.012793 0.069308 -0.097943 0.026051 -0.040273 -0.046429 -0.037232 -0.075387 0.056425 +-0.073314 -0.025474 -0.086713 -0.034135 -0.056786 -0.033425 -0.010800 -0.006306 -0.083626 -0.019529 -0.023848 -0.103377 -0.072124 0.027925 0.038858 0.096703 0.015690 0.004385 0.117400 -0.019385 0.068925 -0.151549 0.006375 -0.118295 0.070831 0.037136 0.042043 -0.099400 -0.017748 0.020165 -0.062059 -0.092214 -0.152389 0.036151 +0.044589 -0.132278 -0.016993 0.083882 0.043778 -0.034594 -0.072011 0.039110 -0.046069 -0.040587 0.123917 -0.047575 -0.122385 -0.035400 -0.024195 -0.012518 0.013652 0.031937 0.019659 -0.001715 0.023088 -0.000309 -0.031436 0.022571 0.019611 -0.006564 0.012784 0.083317 -0.104572 0.062104 0.000647 -0.056669 0.044672 0.004338 +0.016329 -0.109007 -0.005045 0.067630 0.019108 -0.002645 -0.071902 0.025076 0.007237 -0.051092 0.091200 -0.100586 -0.101496 -0.065336 -0.015376 0.013166 0.008475 0.063623 0.047215 0.020495 -0.013691 0.030534 0.016318 -0.010397 -0.023315 0.035663 -0.023907 0.042304 -0.100402 0.062642 -0.013752 0.012255 0.032336 0.012548 +0.005789 -0.090540 0.006017 0.048376 -0.000562 -0.003770 -0.064679 0.019118 0.018765 -0.056858 0.072645 -0.105570 -0.099008 -0.063197 -0.036400 0.020176 0.006068 0.073862 0.022493 -0.014200 -0.022956 0.011319 0.028866 -0.059130 0.024068 0.043375 -0.037957 0.009386 -0.070164 0.052847 -0.054076 0.043887 0.014360 -0.001251 +0.001863 -0.067279 -0.004424 0.022537 -0.021339 0.004205 -0.062496 0.031100 0.053349 -0.011934 0.065752 -0.132376 -0.118604 -0.084354 0.018198 0.004240 -0.004340 0.003937 0.017652 -0.050848 0.024164 -0.013728 -0.034087 -0.019119 0.077051 0.006899 0.013968 -0.043247 -0.030695 0.066883 -0.075176 -0.002876 0.013993 -0.016796 +0.005001 -0.090158 -0.011976 0.055472 -0.011104 -0.019645 -0.061557 0.044266 0.027270 -0.030345 0.068012 -0.132701 -0.097672 -0.062167 -0.026502 -0.015816 -0.002765 0.014242 0.048233 0.014931 0.013361 0.021076 -0.002662 -0.047362 0.074697 -0.011056 0.032939 0.005948 -0.057100 0.111103 -0.023048 -0.033487 -0.004465 0.045128 +0.012182 -0.107450 -0.027021 0.065245 0.008961 -0.027772 -0.074458 0.050282 0.025229 -0.041151 0.086397 -0.119923 -0.099702 -0.055513 -0.011844 -0.012761 -0.000561 0.025024 0.077500 0.042190 0.004065 0.022608 -0.020626 -0.014056 0.026667 -0.025100 0.039847 0.066569 -0.072695 0.106463 0.007401 -0.072767 0.005736 0.071139 +-0.009462 0.065949 -0.034941 0.008894 -0.064655 -0.020166 -0.060698 0.028627 0.066320 0.002801 0.040050 0.031191 -0.045838 0.048249 0.022311 -0.088733 0.012498 0.012592 -0.012528 -0.038889 0.028963 -0.007368 -0.006653 0.082948 -0.006713 0.005645 -0.025498 -0.054304 -0.022462 0.175263 -0.039024 -0.042552 -0.094353 -0.013449 +0.000399 0.075432 -0.010525 0.041532 -0.020281 -0.016042 -0.094246 0.065468 0.057754 -0.013075 0.035003 0.037001 -0.063322 0.051499 0.004328 -0.061380 -0.015256 0.001025 -0.051518 0.003395 -0.032991 0.024094 0.008757 0.054130 -0.053594 0.010905 -0.022832 -0.025114 -0.048340 0.149929 -0.021867 0.081009 -0.007931 -0.009067 +0.011378 0.085467 -0.017111 0.041300 -0.001692 -0.039680 -0.104984 0.079709 0.063221 -0.028397 0.048561 0.041353 -0.084814 0.040411 0.000268 -0.038789 -0.000559 -0.008586 -0.044596 0.040237 -0.063474 -0.003204 -0.031149 0.016331 -0.050858 0.018453 -0.008099 -0.080392 -0.015774 0.091864 -0.016377 0.134277 0.049921 -0.027301 +0.029766 0.102153 -0.038143 0.053924 0.038638 -0.061103 -0.101860 0.090831 0.056649 -0.017724 0.060045 0.040849 -0.108462 0.025297 0.013784 -0.048798 0.017389 -0.036169 -0.049864 0.021916 -0.066484 0.016243 -0.106255 -0.005066 -0.033200 -0.008221 0.050207 -0.137478 0.022810 0.034341 -0.021252 0.120671 0.042481 -0.025859 +0.006889 0.087347 -0.040353 0.029840 -0.010245 -0.040820 -0.091202 0.072685 0.079209 -0.016224 0.043485 -0.001910 -0.084857 0.044870 -0.001342 -0.071705 -0.002282 -0.026485 -0.039688 0.033741 -0.055799 -0.013397 -0.029281 0.000374 -0.037317 -0.004296 0.042230 -0.104897 0.024080 0.125291 0.011644 0.079126 -0.013272 0.018850 +-0.000936 0.077182 -0.035678 0.029087 -0.033747 -0.023682 -0.089809 0.060071 0.081219 -0.000659 0.033152 -0.007048 -0.054525 0.054060 -0.010472 -0.078847 -0.000530 -0.009111 -0.033111 0.004942 -0.009483 0.002476 0.026701 0.054112 -0.050766 -0.014615 0.013950 -0.054182 -0.017408 0.176648 0.019946 -0.001332 -0.069141 0.058993 +-0.053340 -0.049736 0.202485 -0.040699 0.016891 0.098603 -0.079165 0.012530 -0.068702 0.013827 -0.018413 0.064039 -0.096530 -0.107456 -0.034620 0.115038 0.176489 0.048911 0.043125 -0.004140 0.021027 -0.096280 0.063451 0.033838 -0.029605 0.024958 -0.044214 -0.018466 -0.082116 -0.029533 0.007344 -0.061482 0.038981 -0.084020 +-0.084313 -0.038019 0.077710 -0.058798 0.076335 0.124114 -0.036329 -0.056135 -0.011941 0.019801 -0.032605 0.049799 -0.014743 -0.022984 -0.023983 0.002739 0.115499 0.010250 -0.071667 0.052221 0.028701 -0.018035 0.046885 0.065758 -0.088996 0.058902 0.045509 0.021842 0.012293 -0.058012 -0.054652 -0.028210 -0.045572 0.005237 +-0.092694 -0.031386 -0.029516 -0.066011 0.056512 0.140318 0.017759 -0.055408 0.042409 -0.001841 -0.047881 0.006271 -0.044880 0.010562 -0.008572 -0.032142 0.031237 0.011784 -0.071950 -0.017154 0.021388 0.094393 -0.059446 -0.036018 -0.074495 0.032641 0.092606 0.062712 0.134488 -0.112228 -0.109084 0.090907 -0.032517 0.032689 +-0.086068 -0.007082 -0.045041 -0.060765 0.055270 0.132259 0.009008 -0.042025 0.050070 -0.002958 -0.051816 0.001503 -0.066612 -0.017554 -0.020320 -0.047269 0.007336 0.021029 -0.065278 0.045943 0.031788 0.044150 -0.060233 -0.015205 -0.047170 0.032557 0.110315 0.067668 0.142436 -0.112995 -0.095937 0.043696 -0.053263 0.018166 +-0.089411 0.023741 -0.038912 -0.067964 0.042282 0.140245 0.010622 -0.036806 0.040610 0.006060 -0.017474 0.000163 -0.063926 0.009072 -0.059385 -0.064253 -0.040611 0.016892 -0.098455 0.064721 0.042317 0.071934 -0.068174 -0.023171 -0.006137 0.021292 0.136552 0.086574 0.159669 -0.085905 -0.090037 0.025543 -0.060574 -0.002428 +-0.078575 0.034575 0.031263 -0.073552 0.040647 0.136747 -0.043737 -0.008796 0.032331 0.016926 -0.029196 0.043721 -0.033823 -0.069883 -0.054957 0.000382 0.001785 -0.066324 -0.031587 0.075610 0.002026 0.123617 -0.016946 -0.073108 0.082295 -0.003377 0.160760 0.045913 0.138259 -0.012841 -0.021037 0.019773 -0.081271 0.003220 +-0.033366 0.059208 0.160210 -0.064990 0.028624 0.104300 -0.099598 0.045261 -0.039277 0.065556 -0.102912 0.108074 -0.081503 -0.190293 -0.052187 0.075107 0.008671 -0.121671 0.069010 0.122685 -0.035632 -0.012154 0.063158 -0.139158 0.053250 0.013035 0.078006 -0.006941 0.005533 0.101138 0.135668 0.005174 -0.004429 -0.021124 +-0.092656 0.031357 0.103505 -0.059111 0.068764 0.010312 0.008929 0.034866 0.057992 0.068053 0.014425 0.007778 0.041020 -0.040683 -0.027837 -0.022634 -0.126523 -0.071192 0.073877 0.037044 -0.024998 0.031381 0.000446 -0.063882 0.075465 -0.089846 -0.003293 -0.014744 -0.039966 0.049482 0.125205 -0.021707 -0.015263 0.035730 +-0.111708 0.016302 0.021802 -0.017327 0.098757 -0.032632 0.102916 -0.004106 0.112659 0.043944 0.086722 -0.075135 0.053748 0.008377 0.026036 -0.051027 -0.181615 0.007593 -0.029718 -0.022786 -0.013286 0.004632 0.013051 -0.001664 0.043884 -0.080356 -0.074392 -0.041220 -0.096235 -0.001700 0.105291 -0.079759 -0.034690 -0.007809 +-0.106284 -0.022433 0.016803 -0.009022 0.113956 -0.059454 0.115911 -0.012451 0.125340 0.015307 0.082184 -0.057654 0.037795 0.019407 0.065630 -0.026223 -0.104086 0.015953 -0.052260 -0.022874 -0.001840 -0.000797 -0.004680 0.054162 0.031667 -0.086636 -0.106973 -0.052172 -0.115387 -0.039377 0.098135 -0.063370 -0.010368 -0.058260 +-0.105834 -0.044185 0.028207 -0.010833 0.109847 -0.041639 0.113964 -0.035423 0.117943 0.018785 0.073258 -0.043540 0.048715 0.037758 0.075421 -0.003877 -0.043429 0.013819 -0.039051 -0.084951 -0.018739 0.005618 -0.000400 0.028961 0.006799 -0.081430 -0.141147 -0.095855 -0.105823 -0.040548 0.071096 -0.022040 -0.010334 -0.072087 +-0.085770 -0.066339 0.119689 -0.034493 0.098418 0.002001 0.025266 -0.023246 0.059661 0.027629 0.052453 0.022351 0.015313 -0.007107 -0.005840 0.096765 0.062508 -0.012640 -0.032725 -0.080090 -0.027756 -0.041996 0.055252 0.063300 -0.038983 -0.076443 -0.130005 -0.083137 -0.110099 -0.085295 0.021676 -0.004858 0.062427 -0.049960 +-0.063490 -0.034197 0.223953 -0.071609 0.017273 0.072534 -0.070046 0.008472 -0.059151 0.024538 -0.057458 0.054908 -0.069500 -0.108298 -0.024305 0.086168 0.173404 0.023995 0.062833 0.058041 0.036625 -0.084243 -0.007661 0.049316 -0.069261 0.031241 -0.034048 -0.059035 -0.063512 -0.030367 -0.002864 -0.044970 0.061520 -0.067669 +-0.080995 -0.029362 -0.020969 -0.037152 0.055714 0.092056 0.009933 -0.025303 0.020523 -0.004640 -0.020295 -0.009164 -0.005931 0.018848 0.033915 -0.030364 0.045067 0.004015 0.011173 -0.058507 0.056901 -0.042073 -0.006012 0.046935 -0.107241 0.032777 0.116424 0.038938 0.083070 0.044785 -0.024341 -0.040868 -0.054497 0.021228 +-0.079570 -0.003619 -0.038468 -0.043009 0.061929 0.088254 0.002999 -0.017741 0.034379 -0.004900 -0.032478 -0.019353 -0.018212 -0.005055 0.014637 -0.048472 -0.001520 0.022877 0.006651 -0.017392 0.069312 -0.058844 -0.004646 0.065142 -0.096265 0.026598 0.135522 0.071861 0.091550 0.032257 -0.016765 -0.075337 -0.064651 0.050908 +-0.077037 0.036024 -0.029906 -0.041175 0.045056 0.108172 0.001799 -0.013867 0.031473 0.010155 -0.003491 -0.014995 -0.021891 0.005419 -0.006514 -0.067131 -0.050711 0.002908 -0.017700 0.003424 0.063512 -0.037825 -0.008255 0.061508 -0.030670 0.015575 0.161325 0.090053 0.117788 0.091435 -0.010989 -0.096824 -0.085735 0.003260 +-0.051962 0.043127 0.185753 -0.068488 0.030906 0.083935 -0.070252 0.031192 -0.032752 0.081860 -0.058415 0.100433 -0.075819 -0.159865 -0.059411 0.067034 0.026048 -0.117863 0.048451 0.078422 -0.022589 0.005932 0.109565 -0.086305 0.028208 0.027111 0.068306 -0.018619 -0.000395 0.065584 0.129299 -0.018649 -0.057046 -0.000423 +-0.110771 0.026821 0.039896 -0.026400 0.073464 0.011366 0.088839 0.010905 0.059433 0.027762 0.038964 -0.069547 0.034358 0.017245 0.035428 -0.063296 -0.129262 0.026781 -0.092055 0.071498 -0.050266 0.045878 -0.036457 -0.040249 0.060855 -0.044906 -0.090430 -0.018843 -0.111822 -0.025148 0.069191 -0.000310 -0.018828 -0.067576 +-0.109310 -0.015148 0.033856 -0.022173 0.084027 -0.011325 0.100709 0.004099 0.070556 0.012019 0.035921 -0.046262 0.047837 0.012906 0.059339 -0.023335 -0.082080 0.034241 -0.099980 0.038508 -0.057517 0.048531 -0.049704 -0.033707 0.035725 -0.049554 -0.119832 -0.033233 -0.116399 -0.076705 0.057450 0.017861 0.007758 -0.058416 +-0.105002 -0.041286 0.052806 -0.020899 0.081333 0.007181 0.095687 -0.011183 0.063719 0.009967 0.029662 -0.037522 0.062980 0.038468 0.065127 -0.003635 -0.032995 0.016733 -0.100860 -0.006462 -0.072928 0.073195 -0.048581 -0.059580 0.018777 -0.048728 -0.138328 -0.072245 -0.108275 -0.063238 0.030578 0.066053 0.024802 -0.084883 +# The variances of the components (eigenvalues) of identity or combined identity and expression model +1 +34 +6 +826.213804 695.783596 380.584790 282.861398 209.814481 184.418561 113.076307 104.852653 81.857546 77.095662 71.081802 55.760367 49.883312 39.122009 37.668889 32.916377 31.165932 26.644275 24.233150 21.357573 17.029065 15.432317 13.812689 12.440567 10.213426 9.316734 5.327325 2.526152 1.659451 1.123615 1.015840 0.816193 0.718969 0.582660 diff --git a/lib/local/FaceAnalyser/AU_predictors/main_dynamic_svms.txt b/lib/local/FaceAnalyser/AU_predictors/main_dynamic_svms.txt new file mode 100644 index 0000000..49476e2 --- /dev/null +++ b/lib/local/FaceAnalyser/AU_predictors/main_dynamic_svms.txt @@ -0,0 +1,3 @@ +AUPredictor AU_all_best.txt +PDM In-the-wild_aligned_PDM_68.txt +Triangulation tris_68_full.txt \ No newline at end of file diff --git a/lib/local/FaceAnalyser/AU_predictors/main_static_svms.txt b/lib/local/FaceAnalyser/AU_predictors/main_static_svms.txt new file mode 100644 index 0000000..0ef644a --- /dev/null +++ b/lib/local/FaceAnalyser/AU_predictors/main_static_svms.txt @@ -0,0 +1,3 @@ +AUPredictor AU_all_static.txt +PDM In-the-wild_aligned_PDM_68.txt +Triangulation tris_68_full.txt \ No newline at end of file diff --git a/lib/local/FaceAnalyser/AU_predictors/tris_68_full.txt b/lib/local/FaceAnalyser/AU_predictors/tris_68_full.txt new file mode 100644 index 0000000..25ee5c0 --- /dev/null +++ b/lib/local/FaceAnalyser/AU_predictors/tris_68_full.txt @@ -0,0 +1,114 @@ +111 +3 +4 +58 67 59 +60 49 48 +58 59 6 +34 52 35 +44 45 25 +39 40 29 +37 18 36 +27 42 22 +23 44 24 +41 36 1 +50 62 51 +56 65 66 +57 58 7 +64 53 63 +28 27 39 +52 34 51 +54 14 35 +29 42 28 +19 20 24 +35 15 46 +37 19 18 +36 0 1 +18 17 36 +37 20 19 +38 20 37 +21 20 38 +21 38 39 +24 44 25 +30 34 35 +21 39 27 +28 42 27 +39 29 28 +29 30 35 +31 30 29 +30 33 34 +31 29 40 +36 17 0 +41 31 40 +31 32 30 +31 41 1 +49 31 48 +48 2 3 +67 60 59 +4 48 3 +5 48 4 +6 59 5 +59 48 5 +60 48 59 +7 58 6 +61 49 60 +58 66 67 +31 2 48 +31 50 32 +1 2 31 +61 50 49 +52 62 63 +50 31 49 +34 33 51 +51 62 52 +32 50 51 +50 61 62 +63 53 52 +54 55 11 +57 8 9 +66 58 57 +8 57 7 +56 57 9 +66 57 56 +10 56 9 +55 56 10 +53 54 35 +53 35 52 +12 54 11 +55 10 11 +65 56 55 +64 55 54 +65 55 64 +54 53 64 +12 13 54 +14 54 13 +15 35 14 +47 35 46 +33 32 51 +30 32 33 +29 35 47 +15 45 46 +22 21 27 +20 21 23 +43 23 22 +29 47 42 +23 21 22 +24 20 23 +22 42 43 +23 43 44 +45 16 26 +15 16 45 +25 45 26 +41 36 37 +40 41 37 +38 40 37 +39 40 38 +47 42 43 +46 47 43 +46 43 44 +45 46 44 +67 60 61 +66 67 61 +62 66 61 +65 66 62 +63 65 62 +64 65 63 \ No newline at end of file diff --git a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj index 8ac84ae..fa5e3f7 100644 --- a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj +++ b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj @@ -186,18 +186,24 @@ + + + + CppCode + + diff --git a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj.filters b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj.filters index 3edc5a6..84218c8 100644 --- a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj.filters +++ b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj.filters @@ -36,6 +36,15 @@ Header Files + + Header Files + + + Header Files + + + Header Files + @@ -59,5 +68,14 @@ Source Files + + Source Files + + + Source Files + + + Source Files + \ No newline at end of file diff --git a/lib/local/FaceAnalyser/include/FaceAnalyser.h b/lib/local/FaceAnalyser/include/FaceAnalyser.h index 153ea57..cae0b22 100644 --- a/lib/local/FaceAnalyser/include/FaceAnalyser.h +++ b/lib/local/FaceAnalyser/include/FaceAnalyser.h @@ -35,198 +35,214 @@ #ifndef __FACEANALYSER_h_ #define __FACEANALYSER_h_ +// STL includes +#include +#include +#include + +// OpenCV includes +#include + +// Local includes #include "SVR_dynamic_lin_regressors.h" #include "SVR_static_lin_regressors.h" #include "SVM_static_lin.h" #include "SVM_dynamic_lin.h" - -#include -#include - -#include - -#include "LandmarkCoreIncludes.h" +#include "PDM.h" +#include "FaceAnalyserParameters.h" namespace FaceAnalysis { - class FaceAnalyser { +class FaceAnalyser{ - public: +public: - enum RegressorType { SVR_appearance_static_linear = 0, SVR_appearance_dynamic_linear = 1, SVR_dynamic_geom_linear = 2, SVR_combined_linear = 3, SVM_linear_stat = 4, SVM_linear_dyn = 5, SVR_linear_static_seg = 6, SVR_linear_dynamic_seg = 7 }; + enum RegressorType{ SVR_appearance_static_linear = 0, SVR_appearance_dynamic_linear = 1, SVR_dynamic_geom_linear = 2, SVR_combined_linear = 3, SVM_linear_stat = 4, SVM_linear_dyn = 5, SVR_linear_static_seg = 6, SVR_linear_dynamic_seg =7}; - // Constructor from a model file (or a default one if not provided - // TODO scale width and height should be read in as part of the model as opposed to being here? - FaceAnalyser(vector orientation_bins = vector(), double scale = 0.7, int width = 112, int height = 112, std::string au_location = "AU_predictors/AU_all_best.txt", std::string tri_location = "model/tris_68_full.txt"); + // Constructor for FaceAnalyser using the parameters structure + FaceAnalyser(const FaceAnalysis::FaceAnalyserParameters& face_analyser_params); - void AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CLNF& clnf, double timestamp_seconds, bool online = false, bool visualise = true); + void AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online = false, bool visualise = true); - // If the features are extracted manually (shouldn't really be used) - void PredictAUs(const cv::Mat_& hog_features, const cv::Mat_& geom_features, const LandmarkDetector::CLNF& clnf_model, bool online); + cv::Mat GetLatestHOGDescriptorVisualisation(); - cv::Mat GetLatestHOGDescriptorVisualisation(); + double GetCurrentTimeSeconds(); + + // Grab the current predictions about AUs from the face analyser + std::vector> GetCurrentAUsClass() const; // AU presence + std::vector> GetCurrentAUsReg() const; // AU intensity + std::vector> GetCurrentAUsCombined() const; // Both presense and intensity - double GetCurrentTimeSeconds(); + // A standalone call for predicting AUs from a static image, the first element in the pair represents occurence the second intensity + // This call is useful for detecting action units in images + std::pair>, std::vector>> PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise = true); - // Grab the current predictions about AUs from the face analyser - std::vector> GetCurrentAUsClass() const; // AU presence - std::vector> GetCurrentAUsReg() const; // AU intensity - std::vector> GetCurrentAUsCombined() const; // Both presense and intensity + void Reset(); - // A standalone call for predicting AUs from a static image, the first element in the pair represents occurence the second intensity - // This call is useful for detecting action units in images - std::pair>, std::vector>> PredictStaticAUs(const cv::Mat& frame, const LandmarkDetector::CLNF& clnf, bool visualise = true); + void GetLatestHOG(cv::Mat_& hog_descriptor, int& num_rows, int& num_cols); + void GetLatestAlignedFace(cv::Mat& image); + + void GetLatestNeutralHOG(cv::Mat_& hog_descriptor, int& num_rows, int& num_cols); + + cv::Mat_ GetTriangulation(); + + void GetGeomDescriptor(cv::Mat_& geom_desc); - void Reset(); + // Grab the names of AUs being predicted + std::vector GetAUClassNames() const; // Presence + std::vector GetAURegNames() const; // Intensity - void GetLatestHOG(cv::Mat_& hog_descriptor, int& num_rows, int& num_cols); - void GetLatestAlignedFace(cv::Mat& image); - - void GetLatestNeutralHOG(cv::Mat_& hog_descriptor, int& num_rows, int& num_cols); - - cv::Mat_ GetTriangulation(); - - void GetGeomDescriptor(cv::Mat_& geom_desc); - - // Grab the names of AUs being predicted - std::vector GetAUClassNames() const; // Presence - std::vector GetAURegNames() const; // Intensity - - // Identify if models are static or dynamic (useful for correction and shifting) - std::vector GetDynamicAUClass() const; // Presence - std::vector> GetDynamicAUReg() const; // Intensity + // Identify if models are static or dynamic (useful for correction and shifting) + std::vector GetDynamicAUClass() const; // Presence + std::vector> GetDynamicAUReg() const; // Intensity - void ExtractAllPredictionsOfflineReg(vector>>& au_predictions, vector& confidences, vector& successes, vector& timestamps, bool dynamic); - void ExtractAllPredictionsOfflineClass(vector>>& au_predictions, vector& confidences, vector& successes, vector& timestamps, bool dynamic); + void ExtractAllPredictionsOfflineReg(std::vector>>& au_predictions, + std::vector& confidences, std::vector& successes, std::vector& timestamps, bool dynamic); + void ExtractAllPredictionsOfflineClass(std::vector>>& au_predictions, + std::vector& confidences, std::vector& successes, std::vector& timestamps, bool dynamic); - // Helper function for post-processing AU output files - void PostprocessOutputFile(string output_file, bool dynamic); + // Helper function for post-processing AU output files + void PostprocessOutputFile(std::string output_file); - private: +private: - // Where the predictions are kept - std::vector> AU_predictions_reg; - std::vector> AU_predictions_class; + // Point distribution model coddesponding to the current Face Analyser + FaceAnalysis::PDM pdm; - std::vector> AU_predictions_combined; + // Where the predictions are kept + std::vector> AU_predictions_reg; + std::vector> AU_predictions_class; - // Keeping track of AU predictions over time (useful for post-processing) - vector timestamps; - std::map> AU_predictions_reg_all_hist; - std::map> AU_predictions_class_all_hist; - std::vector confidences; - std::vector valid_preds; + std::vector> AU_predictions_combined; - int frames_tracking; + // Keeping track of AU predictions over time (useful for post-processing) + std::vector timestamps; + std::map> AU_predictions_reg_all_hist; + std::map> AU_predictions_class_all_hist; + std::vector valid_preds; - // Cache of intermediate images - cv::Mat aligned_face_for_au; - cv::Mat aligned_face_for_output; - cv::Mat hog_descriptor_visualisation; + int frames_tracking; - // Private members to be used for predictions - // The HOG descriptor of the last frame - cv::Mat_ hog_desc_frame; - int num_hog_rows; - int num_hog_cols; + // Is the AU model dynamic + bool dynamic; - // Keep a running median of the hog descriptors and a aligned images - cv::Mat_ hog_desc_median; - cv::Mat_ face_image_median; + // Cache of intermediate images + cv::Mat aligned_face_for_au; + cv::Mat aligned_face_for_output; + cv::Mat hog_descriptor_visualisation; + bool out_grayscale; - // Use histograms for quick (but approximate) median computation - // Use the same for - vector > hog_desc_hist; + // Private members to be used for predictions + // The HOG descriptor of the last frame + cv::Mat_ hog_desc_frame; + int num_hog_rows; + int num_hog_cols; - // This is not being used at the moment as it is a bit slow - vector > face_image_hist; - vector face_image_hist_sum; + // Keep a running median of the hog descriptors and a aligned images + cv::Mat_ hog_desc_median; + cv::Mat_ face_image_median; - vector head_orientations; + // Use histograms for quick (but approximate) median computation + // Use the same for + std::vector > hog_desc_hist; - int num_bins_hog; - double min_val_hog; - double max_val_hog; - vector hog_hist_sum; - int view_used; + // This is not being used at the moment as it is a bit slow + std::vector > face_image_hist; + std::vector face_image_hist_sum; - // The geometry descriptor (rigid followed by non-rigid shape parameters from CLNF) - cv::Mat_ geom_descriptor_frame; - cv::Mat_ geom_descriptor_median; + std::vector head_orientations; - int geom_hist_sum; - cv::Mat_ geom_desc_hist; - int num_bins_geom; - double min_val_geom; - double max_val_geom; + int num_bins_hog; + double min_val_hog; + double max_val_hog; + std::vector hog_hist_sum; + int view_used; - // Using the bounding box of previous analysed frame to determine if a reset is needed - cv::Rect_ face_bounding_box; + // The geometry descriptor (rigid followed by non-rigid shape parameters from CLNF) + cv::Mat_ geom_descriptor_frame; + cv::Mat_ geom_descriptor_median; + + int geom_hist_sum; + cv::Mat_ geom_desc_hist; + int num_bins_geom; + double min_val_geom; + double max_val_geom; + + // Using the bounding box of previous analysed frame to determine if a reset is needed + cv::Rect_ face_bounding_box; + + // The AU predictions internally + std::vector> PredictCurrentAUs(int view); + std::vector> PredictCurrentAUsClass(int view); - // The AU predictions internally - std::vector> PredictCurrentAUs(int view); - std::vector> PredictCurrentAUsClass(int view); + // special step for online (rather than offline AU prediction) + std::vector> CorrectOnlineAUs(std::vector> predictions_orig, int view, bool dyn_shift = false, bool dyn_scale = false, bool update_track = true, bool clip_values = false); - // special step for online (rather than offline AU prediction) - std::vector> CorrectOnlineAUs(std::vector> predictions_orig, int view, bool dyn_shift = false, bool dyn_scale = false, bool update_track = true, bool clip_values = false); + void Read(std::string model_loc); - void ReadAU(std::string au_location); + void ReadAU(std::string au_location); - void ReadRegressor(std::string fname, const vector& au_names); + void ReadRegressor(std::string fname, const std::vector& au_names); - // A utility function for keeping track of approximate running medians used for AU and emotion inference using a set of histograms (the histograms are evenly spaced from min_val to max_val) - // Descriptor has to be a row vector - // TODO this duplicates some other code - void UpdateRunningMedian(cv::Mat_& histogram, int& hist_sum, cv::Mat_& median, const cv::Mat_& descriptor, bool update, int num_bins, double min_val, double max_val); - void ExtractMedian(cv::Mat_& histogram, int hist_count, cv::Mat_& median, int num_bins, double min_val, double max_val); + // A utility function for keeping track of approximate running medians used for AU and emotion inference using a set of histograms (the histograms are evenly spaced from min_val to max_val) + // Descriptor has to be a row vector + // TODO this duplicates some other code + void UpdateRunningMedian(cv::Mat_& histogram, int& hist_sum, cv::Mat_& median, const cv::Mat_& descriptor, bool update, int num_bins, double min_val, double max_val); + void ExtractMedian(cv::Mat_& histogram, int hist_count, cv::Mat_& median, int num_bins, double min_val, double max_val); + + // The linear SVR regressors + SVR_static_lin_regressors AU_SVR_static_appearance_lin_regressors; + SVR_dynamic_lin_regressors AU_SVR_dynamic_appearance_lin_regressors; + + // The linear SVM classifiers + SVM_static_lin AU_SVM_static_appearance_lin; + SVM_dynamic_lin AU_SVM_dynamic_appearance_lin; - // The linear SVR regressors - SVR_static_lin_regressors AU_SVR_static_appearance_lin_regressors; - SVR_dynamic_lin_regressors AU_SVR_dynamic_appearance_lin_regressors; + // The AUs predicted by the model are not always 0 calibrated to a person. That is they don't always predict 0 for a neutral expression + // Keeping track of the predictions we can correct for this, by assuming that at least "ratio" of frames are neutral and subtract that value of prediction, only perform the correction after min_frames + void UpdatePredictionTrack(cv::Mat_& prediction_corr_histogram, int& prediction_correction_count, + std::vector& correction, const std::vector>& predictions, double ratio=0.25, int num_bins = 200, double min_val = -3, double max_val = 5, int min_frames = 10); + void GetSampleHist(cv::Mat_& prediction_corr_histogram, int prediction_correction_count, + std::vector& sample, double ratio, int num_bins = 200, double min_val = 0, double max_val = 5); - // The linear SVM classifiers - SVM_static_lin AU_SVM_static_appearance_lin; - SVM_dynamic_lin AU_SVM_dynamic_appearance_lin; + void PostprocessPredictions(); - // The AUs predicted by the model are not always 0 calibrated to a person. That is they don't always predict 0 for a neutral expression - // Keeping track of the predictions we can correct for this, by assuming that at least "ratio" of frames are neutral and subtract that value of prediction, only perform the correction after min_frames - void UpdatePredictionTrack(cv::Mat_& prediction_corr_histogram, int& prediction_correction_count, vector& correction, const vector>& predictions, double ratio = 0.25, int num_bins = 200, double min_val = -3, double max_val = 5, int min_frames = 10); - void GetSampleHist(cv::Mat_& prediction_corr_histogram, int prediction_correction_count, vector& sample, double ratio, int num_bins = 200, double min_val = 0, double max_val = 5); + std::vector> au_prediction_correction_histogram; + std::vector au_prediction_correction_count; - void PostprocessPredictions(); + // Some dynamic scaling (the logic is that before the extreme versions of expression or emotion are shown, + // it is hard to tell the boundaries, this allows us to scale the model to the most extreme seen) + // They have to be view specific + std::vector> dyn_scaling; + + // Keeping track of predictions for summary stats + cv::Mat_ AU_prediction_track; + cv::Mat_ geom_desc_track; - vector> au_prediction_correction_histogram; - vector au_prediction_correction_count; + double current_time_seconds; - // Some dynamic scaling (the logic is that before the extreme versions of expression or emotion are shown, - // it is hard to tell the boundaries, this allows us to scale the model to the most extreme seen) - // They have to be view specific - vector> dyn_scaling; + // Used for face alignment + cv::Mat_ triangulation; + double align_scale_au; + int align_width_au; + int align_height_au; - // Keeping track of predictions for summary stats - cv::Mat_ AU_prediction_track; - cv::Mat_ geom_desc_track; + double align_scale_out; + int align_width_out; + int align_height_out; - double current_time_seconds; + // Useful placeholder for renormalizing the initial frames of shorter videos + int max_init_frames = 3000; + std::vector> hog_desc_frames_init; + std::vector> geom_descriptor_frames_init; + std::vector views; + bool postprocessed = false; + int frames_tracking_succ = 0; - // Used for face alignment - cv::Mat_ triangulation; - double align_scale; - int align_width; - int align_height; - - // Useful placeholder for renormalizing the initial frames of shorter videos - int max_init_frames = 3000; - vector> hog_desc_frames_init; - vector> geom_descriptor_frames_init; - vector views; - bool postprocessed = false; - int frames_tracking_succ = 0; - - }; - //=========================================================================== +}; + //=========================================================================== } #endif diff --git a/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h b/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h new file mode 100644 index 0000000..5473225 --- /dev/null +++ b/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h @@ -0,0 +1,100 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +// Parameters of the Face analyser +#ifndef __FACE_ANALYSER_PARAM_H +#define __FACE_ANALYSER_PARAM_H + +#include +#include + +// Boost includes +#include +#include + +using namespace std; + +namespace FaceAnalysis +{ + +struct FaceAnalyserParameters +{ +public: + // Constructors + FaceAnalyserParameters(); + FaceAnalyserParameters(string root_exe); + FaceAnalyserParameters(vector &arguments); + + // These are the parameters of training and will not change and are fixed + const double sim_scale_au = 0.7; + const int sim_size_au = 112; + + // Should the output aligned faces be grayscale + bool grayscale; + + // Use getters and setters for these as they might need to reload models and make sure the scale and size ratio makes sense + void setAlignedOutput(int output_size, double scale=-1); + // This will also change the model location + void OptimizeForVideos(); + void OptimizeForImages(); + + double getSimScaleOut() const { return sim_scale_out; } + int getSimSizeOut() const { return sim_size_out; } + bool getDynamic() const { return dynamic; } + string getModelLoc() const { return string(model_location); } + vector getOrientationBins() const { return vector(orientation_bins); } + +private: + + void init(); + + // Aligned face output size + double sim_scale_out; + int sim_size_out; + + // Should a video stream be assumed + bool dynamic; + + // Where to load the models from + string model_location; + // The location of the executable + boost::filesystem::path root; + + vector orientation_bins; + +}; + +} + +#endif // __FACE_ANALYSER_PARAM_H diff --git a/lib/local/FaceAnalyser/include/Face_utils.h b/lib/local/FaceAnalyser/include/Face_utils.h index 8660800..ee395a6 100644 --- a/lib/local/FaceAnalyser/include/Face_utils.h +++ b/lib/local/FaceAnalyser/include/Face_utils.h @@ -35,19 +35,19 @@ #ifndef __FACE_UTILS_h_ #define __FACE_UTILS_h_ -#include - #include #include +#include "PDM.h" + namespace FaceAnalysis { //=========================================================================== // Defining a set of useful utility functions to be used within FaceAnalyser // Aligning a face to a common reference frame - void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const LandmarkDetector::CLNF& clnf_model, bool rigid = true, double scale = 0.6, int width = 96, int height = 96); - void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const LandmarkDetector::CLNF& clnf_model, const cv::Mat_& triangulation, bool rigid = true, double scale = 0.6, int width = 96, int height = 96); + void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6f params_global, const PDM& pdm, bool rigid = true, float scale = 0.6, int width = 96, int height = 96); + void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6f params_global, const PDM& pdm, const cv::Mat_& triangulation, bool rigid = true, float scale = 0.6, int width = 96, int height = 96); void Extract_FHOG_descriptor(cv::Mat_& descriptor, const cv::Mat& image, int& num_rows, int& num_cols, int cell_size = 8); @@ -57,5 +57,50 @@ namespace FaceAnalysis void ExtractSummaryStatistics(const cv::Mat_& descriptors, cv::Mat_& sum_stats, bool mean, bool stdev, bool max_min); void AddDescriptor(cv::Mat_& descriptors, cv::Mat_ new_descriptor, int curr_frame, int num_frames_to_keep = 120); + //=========================================================================== + // Point set and landmark manipulation functions + //=========================================================================== + // Using Kabsch's algorithm for aligning shapes + //This assumes that align_from and align_to are already mean normalised + cv::Matx22f AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to); + + //============================================================================= + // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other + cv::Matx22f AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst); + + //=========================================================================== + // Visualisation functions + //=========================================================================== + void Project(cv::Mat_& dest, const cv::Mat_& mesh, float fx, float fy, float cx, float cy); + + //=========================================================================== + // Angle representation conversion helpers + //=========================================================================== + cv::Matx33f Euler2RotationMatrix(const cv::Vec3f& eulerAngles); + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + cv::Vec3f RotationMatrix2Euler(const cv::Matx33f& rotation_matrix); + + cv::Vec3f Euler2AxisAngle(const cv::Vec3f& euler); + + cv::Vec3f AxisAngle2Euler(const cv::Vec3f& axis_angle); + + cv::Matx33f AxisAngle2RotationMatrix(const cv::Vec3f& axis_angle); + + cv::Vec3f RotationMatrix2AxisAngle(const cv::Matx33f& rotation_matrix); + + //============================================================================ + // Matrix reading functionality + //============================================================================ + + // Reading a matrix written in a binary format + void ReadMatBin(std::ifstream& stream, cv::Mat &output_mat); + + // Reading in a matrix from a stream + void ReadMat(std::ifstream& stream, cv::Mat& output_matrix); + + // Skipping comments (lines starting with # symbol) + void SkipComments(std::ifstream& stream); + } #endif diff --git a/lib/local/FaceAnalyser/include/PAW.h b/lib/local/FaceAnalyser/include/PAW.h new file mode 100644 index 0000000..968a2ee --- /dev/null +++ b/lib/local/FaceAnalyser/include/PAW.h @@ -0,0 +1,138 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __PAW_h_ +#define __PAW_h_ + +// OpenCV includes +#include + +namespace FaceAnalysis +{ + //=========================================================================== + /** + A Piece-wise Affine Warp + The ideas for this piece-wise affine triangular warping are taken from the + Active appearance models revisited by Iain Matthews and Simon Baker in IJCV 2004 + This is used for both validation of landmark detection, and for avatar animation + + The code is based on the CLM tracker by Jason Saragih et al. + */ + +class PAW{ +public: + // Number of pixels after the warping to neutral shape + int number_of_pixels; + + // Minimum x coordinate in destination + float min_x; + + // minimum y coordinate in destination + float min_y; + + // Destination points (landmarks to be warped to) + cv::Mat_ destination_landmarks; + + // Destination points (landmarks to be warped from) + cv::Mat_ source_landmarks; + + // Triangulation, each triangle is warped using an affine transform + cv::Mat_ triangulation; + + // Triangle index, indicating which triangle each of destination pixels lies in + cv::Mat_ triangle_id; + + // Indicating if the destination warped pixels is valid (lies within a face) + cv::Mat_ pixel_mask; + + // A number of precomputed coefficients that are helpful for quick warping + + // affine coefficients for all triangles (see Matthews and Baker 2004) + // 6 coefficients for each triangle (are computed from alpha and beta) + // This is computed during each warp based on source landmarks + cv::Mat_ coefficients; + + // matrix of (c,x,y) coeffs for alpha + cv::Mat_ alpha; + + // matrix of (c,x,y) coeffs for alpha + cv::Mat_ beta; + + // x-source of warped points + cv::Mat_ map_x; + + // y-source of warped points + cv::Mat_ map_y; + + // Default constructor + PAW(){;} + + // Construct a warp from a destination shape and triangulation + PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation); + + // The final optional argument allows for optimisation if the triangle indices from previous frame are known (for tracking in video) + PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, float in_min_x, float in_min_y, float in_max_x, float in_max_y); + + // Copy constructor + PAW(const PAW& other); + + void Read(std::ifstream &s); + + // The actual warping + void Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp); + + // Compute coefficients needed for warping + void CalcCoeff(); + + // Perform the actual warping + void WarpRegion(cv::Mat_& map_x, cv::Mat_& map_y); + + inline int NumberOfLandmarks() const {return destination_landmarks.rows/2;} ; + inline int NumberOfTriangles() const {return triangulation.rows;} ; + + // The width and height of the warped image + inline int constWidth() const {return pixel_mask.cols;} + inline int Height() const {return pixel_mask.rows;} + +private: + + // Helper functions for dealing with triangles + static bool sameSide(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3); + static bool pointInTriangle(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3); + static int findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess = -1); + + }; + //=========================================================================== +} +#endif diff --git a/lib/local/FaceAnalyser/include/PDM.h b/lib/local/FaceAnalyser/include/PDM.h new file mode 100644 index 0000000..7881869 --- /dev/null +++ b/lib/local/FaceAnalyser/include/PDM.h @@ -0,0 +1,105 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __PDMA_h_ +#define __PDMA_h_ + +// OpenCV includes +#include + +#include +#include +#include + +namespace FaceAnalysis +{ +//=========================================================================== +// A linear 3D Point Distribution Model (constructed using Non-Rigid structure from motion or PCA) +// Only describes the model but does not contain an instance of it (no local or global parameters are stored here) +// Contains the utility functions to help manipulate the model + +class PDM{ + public: + + // The 3D mean shape vector of the PDM [x1,..,xn,y1,...yn,z1,...,zn] + cv::Mat_ mean_shape; + + // Principal components or variation bases of the model, + cv::Mat_ princ_comp; + + // Eigenvalues (variances) corresponding to the bases + cv::Mat_ eigen_values; + + PDM(){;} + + // A copy constructor + PDM(const PDM& other); + + void Read(std::string location); + + // Number of vertices + inline int NumberOfPoints() const {return mean_shape.rows/3;} + + // Listing the number of modes of variation + inline int NumberOfModes() const {return princ_comp.cols;} + + // Compute shape in object space (3D) + void CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& params_local) const; + + // Compute shape in image space (2D) + void CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6f& params_global) const; + + // provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box + void CalcParams(cv::Vec6f& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3f rotation = cv::Vec3f(0.0f)) const; + + // Provided the landmark location compute global and local parameters best fitting it (can provide optional rotation for potentially better results) + void CalcParams(cv::Vec6f& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3f rotation = cv::Vec3f(0.0f)) const; + + // provided the model parameters, compute the bounding box of a face + void CalcBoundingBox(cv::Rect_& out_bounding_box, const cv::Vec6f& params_global, const cv::Mat_& params_local) const; + + // Helpers for computing Jacobians, and Jacobians with the weight matrix + void ComputeRigidJacobian(const cv::Mat_& params_local, const cv::Vec6f& params_global, cv::Mat_ &Jacob, const cv::Mat_ W, cv::Mat_ &Jacob_t_w) const; + void ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6f& params_global, cv::Mat_ &Jacobian, const cv::Mat_ W, cv::Mat_ &Jacob_t_w) const; + + // Given the current parameters, and the computed delta_p compute the updated parameters + void UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& params_local, cv::Vec6f& params_global) const; + + // Helper utilities + private: + static void Orthonormalise(cv::Matx33f &R); + }; + //=========================================================================== +} +#endif diff --git a/lib/local/FaceAnalyser/include/SVM_dynamic_lin.h b/lib/local/FaceAnalyser/include/SVM_dynamic_lin.h index 39fdd6a..5b80134 100644 --- a/lib/local/FaceAnalyser/include/SVM_dynamic_lin.h +++ b/lib/local/FaceAnalyser/include/SVM_dynamic_lin.h @@ -40,6 +40,7 @@ #include #include +#include #include diff --git a/lib/local/FaceAnalyser/include/SVM_static_lin.h b/lib/local/FaceAnalyser/include/SVM_static_lin.h index ea1b0ed..3d58d2a 100644 --- a/lib/local/FaceAnalyser/include/SVM_static_lin.h +++ b/lib/local/FaceAnalyser/include/SVM_static_lin.h @@ -40,6 +40,7 @@ #include #include +#include #include diff --git a/lib/local/FaceAnalyser/include/SVR_dynamic_lin_regressors.h b/lib/local/FaceAnalyser/include/SVR_dynamic_lin_regressors.h index 6de0bfe..b90b9a3 100644 --- a/lib/local/FaceAnalyser/include/SVR_dynamic_lin_regressors.h +++ b/lib/local/FaceAnalyser/include/SVR_dynamic_lin_regressors.h @@ -40,6 +40,7 @@ #include #include +#include #include diff --git a/lib/local/FaceAnalyser/include/SVR_static_lin_regressors.h b/lib/local/FaceAnalyser/include/SVR_static_lin_regressors.h index 14a6fd6..8f4fc3e 100644 --- a/lib/local/FaceAnalyser/include/SVR_static_lin_regressors.h +++ b/lib/local/FaceAnalyser/include/SVR_static_lin_regressors.h @@ -40,6 +40,7 @@ #include #include +#include #include diff --git a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp index e91afc6..3702537 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp @@ -41,6 +41,7 @@ // System includes #include #include +#include #include @@ -51,7 +52,6 @@ #include // Local includes -#include "LandmarkCoreIncludes.h" #include "Face_utils.h" using namespace FaceAnalysis; @@ -59,13 +59,17 @@ using namespace FaceAnalysis; using namespace std; // Constructor from a model file (or a default one if not provided -FaceAnalyser::FaceAnalyser(vector orientation_bins, double scale, int width, int height, std::string au_location, std::string tri_location) +FaceAnalyser::FaceAnalyser(const FaceAnalysis::FaceAnalyserParameters& face_analyser_params) { - this->ReadAU(au_location); + this->Read(face_analyser_params.getModelLoc()); - align_scale = scale; - align_width = width; - align_height = height; + align_scale_out = face_analyser_params.getSimScaleOut(); + align_width_out = face_analyser_params.getSimSizeOut(); + align_height_out = face_analyser_params.getSimSizeOut(); + + align_scale_au = face_analyser_params.sim_scale_au; + align_width_au = face_analyser_params.sim_size_au; + align_height_au = face_analyser_params.sim_size_au; // Initialise the histograms that will represent bins from 0 - 1 (as HoG values are only stored as those) num_bins_hog = 1000; @@ -79,15 +83,20 @@ FaceAnalyser::FaceAnalyser(vector orientation_bins, double scale, int // Keep track for how many frames have been tracked so far frames_tracking = 0; - - if(orientation_bins.empty()) + + // If the model used is dynamic (person callibration and video correction) + dynamic = face_analyser_params.getDynamic(); + + out_grayscale = face_analyser_params.grayscale; + + if(face_analyser_params.getOrientationBins().empty()) { // Just using frontal currently head_orientations.push_back(cv::Vec3d(0,0,0)); } else { - head_orientations = orientation_bins; + head_orientations = face_analyser_params.getOrientationBins(); } hog_hist_sum.resize(head_orientations.size()); face_image_hist_sum.resize(head_orientations.size()); @@ -99,10 +108,6 @@ FaceAnalyser::FaceAnalyser(vector orientation_bins, double scale, int au_prediction_correction_histogram.resize(head_orientations.size()); dyn_scaling.resize(head_orientations.size()); - // The triangulation used for masking out the non-face parts of aligned image - std::ifstream triangulation_file(tri_location); - LandmarkDetector::ReadMat(triangulation_file, triangulation); - } // Utility for getting the names of returned AUs (presence) @@ -243,12 +248,17 @@ int GetViewId(const vector orientations_all, const cv::Vec3d& orienta } -std::pair>, std::vector>> FaceAnalyser::PredictStaticAUs(const cv::Mat& frame, const LandmarkDetector::CLNF& clnf, bool visualise) +std::pair>, std::vector>> FaceAnalyser::PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise) { + + // Extract shape parameters from the detected landmarks + cv::Vec6f params_global; + cv::Mat_ params_local; + pdm.CalcParams(params_global, params_local, detected_landmarks); // First align the face - AlignFaceMask(aligned_face_for_au, frame, clnf, triangulation, true, 0.7, 112, 112); - + AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, 0.7, 112, 112); + // Extract HOG descriptor from the frame and convert it to a useable format cv::Mat_ hog_descriptor; Extract_FHOG_descriptor(hog_descriptor, aligned_face_for_au, this->num_hog_rows, this->num_hog_cols); @@ -256,17 +266,21 @@ std::pair>, std::vectorhead_orientations, curr_orient); - - // Geom descriptor and its median - geom_descriptor_frame = clnf.params_local.t(); + + // Geom descriptor and its median, TODO these should be floats? + params_local = params_local.t(); + params_local.convertTo(geom_descriptor_frame, CV_64F); + + cv::Mat_ princ_comp_d; + pdm.princ_comp.convertTo(princ_comp_d, CV_64F); // Stack with the actual feature point locations (without mean) - cv::Mat_ locs = clnf.pdm.princ_comp * geom_descriptor_frame.t(); + cv::Mat_ locs = princ_comp_d * geom_descriptor_frame.t(); cv::hconcat(locs.t(), geom_descriptor_frame.clone(), geom_descriptor_frame); - + // First convert the face image to double representation as a row vector, TODO rem //cv::Mat_ aligned_face_cols(1, aligned_face_for_au.cols * aligned_face_for_au.rows * aligned_face_for_au.channels(), aligned_face_for_au.data, 1); //cv::Mat_ aligned_face_cols_double; @@ -296,44 +310,54 @@ std::pair>, std::vector& detected_landmarks, bool success, double timestamp_seconds, bool online, bool visualise) { frames_tracking++; + // Extract shape parameters from the detected landmarks + cv::Vec6f params_global; + cv::Mat_ params_local; + pdm.CalcParams(params_global, params_local, detected_landmarks); + // First align the face if tracking was successfull - if (clnf_model.detection_success) + if(success) { // The aligned face requirement for AUs - AlignFaceMask(aligned_face_for_au, frame, clnf_model, triangulation, true, 0.7, 112, 112); + AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_au, align_width_au, align_height_au); // If the output requirement matches use the already computed one, else compute it again - if (align_scale == 0.7 && align_width == 112 && align_height == 112) + if(align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au) { aligned_face_for_output = aligned_face_for_au.clone(); } else { - AlignFaceMask(aligned_face_for_output, frame, clnf_model, triangulation, true, align_scale, align_width, align_height); + AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out); } } else { - aligned_face_for_output = cv::Mat(align_height, align_width, CV_8UC3); - aligned_face_for_au = cv::Mat(112, 112, CV_8UC3); + aligned_face_for_output = cv::Mat(align_height_out, align_width_out, CV_8UC3); + aligned_face_for_au = cv::Mat(align_height_au, align_width_au, CV_8UC3); aligned_face_for_output.setTo(0); aligned_face_for_au.setTo(0); } + if (aligned_face_for_output.channels() == 3 && out_grayscale) + { + cvtColor(aligned_face_for_output, aligned_face_for_output, CV_BGR2GRAY); + } + // Extract HOG descriptor from the frame and convert it to a useable format cv::Mat_ hog_descriptor; Extract_FHOG_descriptor(hog_descriptor, aligned_face_for_au, this->num_hog_rows, this->num_hog_cols); - + // Store the descriptor hog_desc_frame = hog_descriptor; - cv::Vec3d curr_orient(clnf_model.params_global[1], clnf_model.params_global[2], clnf_model.params_global[3]); + cv::Vec3d curr_orient(params_global[1], params_global[2], params_global[3]); int orientation_to_use = GetViewId(this->head_orientations, curr_orient); // Only update the running median if predictions are not high @@ -365,39 +389,43 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL // } //} - update_median = update_median & clnf_model.detection_success; + update_median = update_median & success; - if (clnf_model.detection_success) + if (success) frames_tracking_succ++; // A small speedup - if (frames_tracking % 2 == 1) + if(frames_tracking % 2 == 1) { UpdateRunningMedian(this->hog_desc_hist[orientation_to_use], this->hog_hist_sum[orientation_to_use], this->hog_desc_median, hog_descriptor, update_median, this->num_bins_hog, this->min_val_hog, this->max_val_hog); this->hog_desc_median.setTo(0, this->hog_desc_median < 0); - } + } // Geom descriptor and its median - geom_descriptor_frame = clnf_model.params_local.t(); + params_local = params_local.t(); + params_local.convertTo(geom_descriptor_frame, CV_64F); - if (!clnf_model.detection_success) + if(!success) { geom_descriptor_frame.setTo(0); } // Stack with the actual feature point locations (without mean) - cv::Mat_ locs = clnf_model.pdm.princ_comp * geom_descriptor_frame.t(); - + // TODO rem double + cv::Mat_ princ_comp_d; + pdm.princ_comp.convertTo(princ_comp_d, CV_64F); + cv::Mat_ locs = princ_comp_d * geom_descriptor_frame.t(); + cv::hconcat(locs.t(), geom_descriptor_frame.clone(), geom_descriptor_frame); - + // A small speedup - if (frames_tracking % 2 == 1) + if(frames_tracking % 2 == 1) { UpdateRunningMedian(this->geom_desc_hist, this->geom_hist_sum, this->geom_descriptor_median, geom_descriptor_frame, update_median, this->num_bins_geom, this->min_val_geom, this->max_val_geom); } // Visualising the median HOG - if (visualise) + if(visualise) { FaceAnalysis::Visualise_FHOG(hog_descriptor, num_hog_rows, num_hog_cols, hog_descriptor_visualisation); } @@ -406,9 +434,9 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL AU_predictions_reg = PredictCurrentAUs(orientation_to_use); std::vector> AU_predictions_reg_corrected; - if (online) + if(online) { - AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success, true); + AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, success, true); } // Add the reg predictions to the historic data @@ -417,7 +445,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL // Find the appropriate AU (if not found add it) // Only add if the detection was successful - if (clnf_model.detection_success) + if(success) { AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(AU_predictions_reg[au].second); } @@ -426,7 +454,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(0); } } - + AU_predictions_class = PredictCurrentAUsClass(orientation_to_use); for (size_t au = 0; au < AU_predictions_class.size(); ++au) @@ -434,7 +462,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL // Find the appropriate AU (if not found add it) // Only add if the detection was successful - if (clnf_model.detection_success) + if(success) { AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(AU_predictions_class[au].second); } @@ -443,15 +471,15 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(0); } } + - - if (online) + if(online) { AU_predictions_reg = AU_predictions_reg_corrected; } else { - if (clnf_model.detection_success && frames_tracking_succ - 1 < max_init_frames) + if (success && frames_tracking_succ - 1 < max_init_frames) { hog_desc_frames_init.push_back(hog_descriptor); geom_descriptor_frames_init.push_back(geom_descriptor_frame); @@ -462,10 +490,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL this->current_time_seconds = timestamp_seconds; view_used = orientation_to_use; - - bool success = clnf_model.detection_success; - - confidences.push_back(clnf_model.detection_certainty); + valid_preds.push_back(success); timestamps.push_back(timestamp_seconds); @@ -476,79 +501,6 @@ void FaceAnalyser::GetGeomDescriptor(cv::Mat_& geom_desc) geom_desc = this->geom_descriptor_frame.clone(); } -void FaceAnalyser::PredictAUs(const cv::Mat_& hog_features, const cv::Mat_& geom_features, const LandmarkDetector::CLNF& clnf_model, bool online) -{ - // Store the descriptor - hog_desc_frame = hog_features.clone(); - this->geom_descriptor_frame = geom_features.clone(); - - cv::Vec3d curr_orient(clnf_model.params_global[1], clnf_model.params_global[2], clnf_model.params_global[3]); - int orientation_to_use = GetViewId(this->head_orientations, curr_orient); - - // Perform AU prediction - AU_predictions_reg = PredictCurrentAUs(orientation_to_use); - - std::vector> AU_predictions_reg_corrected; - if(online) - { - AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success); - } - - // Add the reg predictions to the historic data - for (size_t au = 0; au < AU_predictions_reg.size(); ++au) - { - - // Find the appropriate AU (if not found add it) - // Only add if the detection was successful - if(clnf_model.detection_success) - { - AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(AU_predictions_reg[au].second); - } - else - { - AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(0.0); - } - } - - AU_predictions_class = PredictCurrentAUsClass(orientation_to_use); - - for (size_t au = 0; au < AU_predictions_class.size(); ++au) - { - - // Find the appropriate AU (if not found add it) - // Only add if the detection was successful - if(clnf_model.detection_success) - { - AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(AU_predictions_class[au].second); - } - else - { - AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(0.0); - } - } - - if(online) - { - AU_predictions_reg = AU_predictions_reg_corrected; - } - - for(size_t i = 0; i < AU_predictions_reg.size(); ++i) - { - AU_predictions_combined.push_back(AU_predictions_reg[i]); - } - for(size_t i = 0; i < AU_predictions_class.size(); ++i) - { - AU_predictions_combined.push_back(AU_predictions_class[i]); - } - - view_used = orientation_to_use; - - bool success = clnf_model.detection_success; - - confidences.push_back(clnf_model.detection_certainty); - valid_preds.push_back(success); -} - // Perform prediction on initial n frames anew as the current neutral face estimate is better now void FaceAnalyser::PostprocessPredictions() { @@ -607,7 +559,6 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector> aus_valid; vector offsets; - confidences = this->confidences; successes = this->valid_preds; vector dyn_au_names = AU_SVR_dynamic_appearance_lin_regressors.GetAUNames(); @@ -654,7 +605,7 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector au_vals_tmp = au_iter->second; - for (int i = (window_size - 1) / 2; i < (int)au_iter->second.size() - (window_size - 1) / 2; ++i) + for (size_t i = (window_size - 1) / 2; i < au_iter->second.size() - (window_size - 1) / 2; ++i) { double sum = 0; - int count_over = 0; for (int w = -(window_size - 1) / 2; w <= (window_size - 1) / 2; ++w) { sum += au_vals_tmp[i + w]; - count_over++; } - sum = sum / count_over; + sum = sum / window_size; au_iter->second[i] = sum; } @@ -736,16 +685,14 @@ void FaceAnalyser::ExtractAllPredictionsOfflineClass(vector au_vals_tmp = au_vals; - for (int i = (window_size - 1)/2; i < (int)au_vals.size() - (window_size - 1) / 2; ++i) + for (size_t i = (window_size - 1)/2; i < au_vals.size() - (window_size - 1) / 2; ++i) { double sum = 0; - int count_over = 0; for (int w = -(window_size - 1) / 2; w <= (window_size - 1) / 2; ++w) { sum += au_vals_tmp[i + w]; - count_over++; } - sum = sum / count_over; + sum = sum / window_size; if (sum < 0.5) sum = 0; else @@ -758,7 +705,6 @@ void FaceAnalyser::ExtractAllPredictionsOfflineClass(vectorconfidences; successes = this->valid_preds; } @@ -801,7 +747,6 @@ void FaceAnalyser::Reset() timestamps.clear(); AU_predictions_reg_all_hist.clear(); AU_predictions_class_all_hist.clear(); - confidences.clear(); valid_preds.clear(); // Clean up the postprocessing data as well @@ -1057,10 +1002,78 @@ vector> FaceAnalyser::GetCurrentAUsCombined() const return AU_predictions_combined; } +void FaceAnalyser::Read(std::string model_loc) +{ + // Reading in the modules for AU recognition + + cout << "Reading the AU analysis module from: " << model_loc << endl; + + ifstream locations(model_loc.c_str(), ios_base::in); + if (!locations.is_open()) + { + cout << "Couldn't open the model file, aborting" << endl; + return; + } + string line; + + // The other module locations should be defined as relative paths from the main model + boost::filesystem::path root = boost::filesystem::path(model_loc).parent_path(); + + // The main file contains the references to other files + while (!locations.eof()) + { + getline(locations, line); + + stringstream lineStream(line); + + string module; + string location; + + // figure out which module is to be read from which file + lineStream >> module; + + lineStream >> location; + + // remove carriage return at the end for compatibility with unix systems + if (location.size() > 0 && location.at(location.size() - 1) == '\r') + { + location = location.substr(0, location.size() - 1); + } + + // append to root + location = (root / location).string(); + if (module.compare("AUPredictor") == 0) + { + // The AU predictors + cout << "Reading the AU predictors from: " << location; + ReadAU(location); + cout << "... Done" << endl; + } + else if (module.compare("PDM") == 0) + { + cout << "Reading the PDM from: " << location; + pdm = PDM(); + pdm.Read(location); + cout << "... Done" << endl; + } + else if (module.compare("Triangulation") == 0) + { + cout << "Reading the triangulation from:" << location; + // The triangulation used for masking out the non-face parts of aligned image + std::ifstream triangulation_file(location); + ReadMat(triangulation_file, triangulation); + cout << "... Done" << endl; + } + } + +} + // Reading in AU prediction modules void FaceAnalyser::ReadAU(std::string au_model_location) { + + // Open the list of the regressors in the file ifstream locations(au_model_location.c_str(), ios::in); @@ -1092,7 +1105,7 @@ void FaceAnalyser::ReadAU(std::string au_model_location) // Parse comma separated names that this regressor produces name = lineStream.str(); - int index = name.find_first_of(' '); + int index = (int)name.find_first_of(' '); if(index >= 0) { @@ -1126,13 +1139,13 @@ void FaceAnalyser::UpdatePredictionTrack(cv::Mat_& prediction_corr // The median update if(prediction_corr_histogram.empty()) { - prediction_corr_histogram = cv::Mat_(predictions.size(), num_bins, (unsigned int)0); + prediction_corr_histogram = cv::Mat_((int)predictions.size(), num_bins, (unsigned int)0); } for(int i = 0; i < prediction_corr_histogram.rows; ++i) { // Find the bins corresponding to the current descriptor - int index = (predictions[i].second - min_val)*((double)num_bins)/(length); + int index = (int)((predictions[i].second - min_val)*((double)num_bins)/(length)); if(index < 0) { index = 0; @@ -1150,7 +1163,7 @@ void FaceAnalyser::UpdatePredictionTrack(cv::Mat_& prediction_corr if(prediction_correction_count >= min_frames) { // Recompute the correction - int cutoff_point = ratio * prediction_correction_count; + int cutoff_point = (int)(ratio * prediction_correction_count); // For each dimension for(int i = 0; i < prediction_corr_histogram.rows; ++i) @@ -1180,7 +1193,7 @@ void FaceAnalyser::GetSampleHist(cv::Mat_& prediction_corr_histogr sample.resize(prediction_corr_histogram.rows, 0); // Recompute the correction - int cutoff_point = ratio * prediction_correction_count; + int cutoff_point = (int)(ratio * prediction_correction_count); // For each dimension for(int i = 0; i < prediction_corr_histogram.rows; ++i) @@ -1231,7 +1244,7 @@ double FaceAnalyser::GetCurrentTimeSeconds() { } // Allows for post processing of the AU signal -void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic) +void FaceAnalyser::PostprocessOutputFile(string output_file) { vector certainties; @@ -1244,8 +1257,8 @@ void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic) ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic); ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic); - int num_class = predictions_class.size(); - int num_reg = predictions_reg.size(); + int num_class = (int)predictions_class.size(); + int num_reg = (int)predictions_reg.size(); // Extract the indices of writing out first vector au_reg_names = GetAURegNames(); @@ -1302,7 +1315,7 @@ void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic) { if (tokens[i].find("AU") != string::npos && begin_ind == -1) { - begin_ind = i; + begin_ind = (int)i; break; } } diff --git a/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp b/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp new file mode 100644 index 0000000..05ef007 --- /dev/null +++ b/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp @@ -0,0 +1,254 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "FaceAnalyserParameters.h" + +// System includes +#include +#include +#include + +#ifndef CONFIG_DIR +#define CONFIG_DIR "~" +#endif + +using namespace std; + +using namespace FaceAnalysis; + +FaceAnalyserParameters::FaceAnalyserParameters():root() +{ + // initialise the default values + init(); +} + +FaceAnalyserParameters::FaceAnalyserParameters(string root_dir) +{ + this->root = root_dir; + init(); + +} +FaceAnalyserParameters::FaceAnalyserParameters(vector &arguments):root() +{ + + // First element is reserved for the executable location (useful for finding relative model locs) + this->root = boost::filesystem::path(arguments[0]).parent_path(); + + // initialise the default values + init(); + + bool* valid = new bool[arguments.size()]; + valid[0] = true; + + bool scale_set = false; + bool size_set = false; + + for (size_t i = 1; i < arguments.size(); ++i) + { + valid[i] = true; + + if (arguments[i].compare("-au_static") == 0) + { + dynamic = false; + valid[i] = false; + } + else if (arguments[i].compare("-g") == 0) + { + grayscale = true; + valid[i] = false; + } + else if (arguments[i].compare("-simscale") == 0) + { + sim_scale_out = stod(arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + scale_set = true; + i++; + } + else if (arguments[i].compare("-simsize") == 0) + { + sim_size_out = stoi(arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + size_set = true; + i++; + } + } + + for (int i = (int)arguments.size() - 1; i >= 0; --i) + { + if (!valid[i]) + { + arguments.erase(arguments.begin() + i); + } + } + + if (dynamic) + { + this->model_location = "AU_predictors/main_dynamic_svms.txt"; + } + else + { + this->model_location = "AU_predictors/main_static_svms.txt"; + } + + // If we set the size but not the scale, adapt the scale to the right size + if (!scale_set && size_set) sim_scale_out = sim_size_out * (0.7 / 112.0); + + // Make sure model_location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); + boost::filesystem::path model_path = boost::filesystem::path(this->model_location); + if (boost::filesystem::exists(model_path)) + { + this->model_location = model_path.string(); + } + else if (boost::filesystem::exists(root/model_path)) + { + this->model_location = (root/model_path).string(); + } + else if (boost::filesystem::exists(config_path/model_path)) + { + this->model_location = (config_path/model_path).string(); + } + else + { + std::cout << "Could not find the AU detection model to load" << std::endl; + } +} + +void FaceAnalyserParameters::init() +{ + // Initialize default parameter values + this->dynamic = true; + this->grayscale = false; + this->sim_scale_out = 0.7; + this->sim_size_out = 112; + + this->model_location = "AU_predictors/main_dynamic_svms.txt"; + + // Make sure model_location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); + boost::filesystem::path model_path = boost::filesystem::path(this->model_location); + if (boost::filesystem::exists(model_path)) + { + this->model_location = model_path.string(); + } + else if (boost::filesystem::exists(root / model_path)) + { + this->model_location = (root / model_path).string(); + } + else if (boost::filesystem::exists(config_path / model_path)) + { + this->model_location = (config_path / model_path).string(); + } + else + { + std::cout << "Could not find the AU detection model to load" << std::endl; + } + + orientation_bins = vector(); + +} + +// Use getters and setters for these as they might need to reload models and make sure the scale and size ratio makes sense +void FaceAnalyserParameters::setAlignedOutput(int output_size, double scale) +{ + this->sim_size_out = output_size; + // If we set the size but not the scale, adapt the scale to the right size + if (scale ==-1) this->sim_scale_out = sim_size_out * (0.7 / 112.0); + else this->sim_scale_out = sim_scale_out; + +} +// This will also change the model location +void FaceAnalyserParameters::OptimizeForVideos() +{ + // Set the post-processing to true and load a dynamic model + dynamic = true; + + this->model_location = "AU_predictors/main_dynamic_svms.txt"; + + // Make sure model_location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); + boost::filesystem::path model_path = boost::filesystem::path(this->model_location); + if (boost::filesystem::exists(model_path)) + { + this->model_location = model_path.string(); + } + else if (boost::filesystem::exists(root / model_path)) + { + this->model_location = (root / model_path).string(); + } + else if (boost::filesystem::exists(config_path / model_path)) + { + this->model_location = (config_path / model_path).string(); + } + else + { + std::cout << "Could not find the AU detection model to load" << std::endl; + } + +} + +void FaceAnalyserParameters::OptimizeForImages() +{ + // Set the post-processing to true and load a dynamic model + dynamic = false; + + this->model_location = "AU_predictors/main_static_svms.txt"; + + // Make sure model_location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); + boost::filesystem::path model_path = boost::filesystem::path(this->model_location); + if (boost::filesystem::exists(model_path)) + { + this->model_location = model_path.string(); + } + else if (boost::filesystem::exists(root / model_path)) + { + this->model_location = (root / model_path).string(); + } + else if (boost::filesystem::exists(config_path / model_path)) + { + this->model_location = (config_path / model_path).string(); + } + else + { + std::cout << "Could not find the AU detection model to load" << std::endl; + } +} + diff --git a/lib/local/FaceAnalyser/src/Face_utils.cpp b/lib/local/FaceAnalyser/src/Face_utils.cpp index fd42789..aa2e730 100644 --- a/lib/local/FaceAnalyser/src/Face_utils.cpp +++ b/lib/local/FaceAnalyser/src/Face_utils.cpp @@ -33,13 +33,16 @@ /////////////////////////////////////////////////////////////////////////////// #include +#include // OpenCV includes #include #include +#include // For FHOG visualisation #include +#include using namespace std; @@ -47,12 +50,12 @@ namespace FaceAnalysis { // Pick only the more stable/rigid points under changes of expression - void extract_rigid_points(cv::Mat_& source_points, cv::Mat_& destination_points) + void extract_rigid_points(cv::Mat_& source_points, cv::Mat_& destination_points) { if(source_points.rows == 68) { - cv::Mat_ tmp_source = source_points.clone(); - source_points = cv::Mat_(); + cv::Mat_ tmp_source = source_points.clone(); + source_points = cv::Mat_(); // Push back the rigid points (some face outline, eyes, and nose) source_points.push_back(tmp_source.row(1)); @@ -80,8 +83,8 @@ namespace FaceAnalysis source_points.push_back(tmp_source.row(46)); source_points.push_back(tmp_source.row(47)); - cv::Mat_ tmp_dest = destination_points.clone(); - destination_points = cv::Mat_(); + cv::Mat_ tmp_dest = destination_points.clone(); + destination_points = cv::Mat_(); // Push back the rigid points destination_points.push_back(tmp_dest.row(1)); @@ -112,16 +115,16 @@ namespace FaceAnalysis } // Aligning a face to a common reference frame - void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const LandmarkDetector::CLNF& clnf_model, bool rigid, double sim_scale, int out_width, int out_height) + void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, bool rigid, float sim_scale, int out_width, int out_height) { // Will warp to scaled mean shape - cv::Mat_ similarity_normalised_shape = clnf_model.pdm.mean_shape * sim_scale; + cv::Mat_ similarity_normalised_shape = pdm.mean_shape * sim_scale; // Discard the z component similarity_normalised_shape = similarity_normalised_shape(cv::Rect(0, 0, 1, 2*similarity_normalised_shape.rows/3)).clone(); - cv::Mat_ source_landmarks = clnf_model.detected_landmarks.reshape(1, 2).t(); - cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); + cv::Mat_ source_landmarks = detected_landmarks.reshape(1, 2).t(); + cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); // Aligning only the more rigid points if(rigid) @@ -129,7 +132,8 @@ namespace FaceAnalysis extract_rigid_points(source_landmarks, destination_landmarks); } - cv::Matx22d scale_rot_matrix = LandmarkDetector::AlignShapesWithScale(source_landmarks, destination_landmarks); + // TODO rem the doubles here + cv::Matx22d scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks); cv::Matx23d warp_matrix; warp_matrix(0,0) = scale_rot_matrix(0,0); @@ -137,8 +141,8 @@ namespace FaceAnalysis warp_matrix(1,0) = scale_rot_matrix(1,0); warp_matrix(1,1) = scale_rot_matrix(1,1); - double tx = clnf_model.params_global[4]; - double ty = clnf_model.params_global[5]; + double tx = params_global[4]; + double ty = params_global[5]; cv::Vec2d T(tx, ty); T = scale_rot_matrix * T; @@ -151,16 +155,16 @@ namespace FaceAnalysis } // Aligning a face to a common reference frame - void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const LandmarkDetector::CLNF& clnf_model, const cv::Mat_& triangulation, bool rigid, double sim_scale, int out_width, int out_height) + void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6f params_global, const PDM& pdm, const cv::Mat_& triangulation, bool rigid, float sim_scale, int out_width, int out_height) { // Will warp to scaled mean shape - cv::Mat_ similarity_normalised_shape = clnf_model.pdm.mean_shape * sim_scale; + cv::Mat_ similarity_normalised_shape = pdm.mean_shape * sim_scale; // Discard the z component similarity_normalised_shape = similarity_normalised_shape(cv::Rect(0, 0, 1, 2*similarity_normalised_shape.rows/3)).clone(); - cv::Mat_ source_landmarks = clnf_model.detected_landmarks.reshape(1, 2).t(); - cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); + cv::Mat_ source_landmarks = detected_landmarks.reshape(1, 2).t(); + cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); // Aligning only the more rigid points if(rigid) @@ -168,18 +172,18 @@ namespace FaceAnalysis extract_rigid_points(source_landmarks, destination_landmarks); } - cv::Matx22d scale_rot_matrix = LandmarkDetector::AlignShapesWithScale(source_landmarks, destination_landmarks); - cv::Matx23d warp_matrix; + cv::Matx22f scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks); + cv::Matx23f warp_matrix; warp_matrix(0,0) = scale_rot_matrix(0,0); warp_matrix(0,1) = scale_rot_matrix(0,1); warp_matrix(1,0) = scale_rot_matrix(1,0); warp_matrix(1,1) = scale_rot_matrix(1,1); - double tx = clnf_model.params_global[4]; - double ty = clnf_model.params_global[5]; + float tx = params_global[4]; + float ty = params_global[5]; - cv::Vec2d T(tx, ty); + cv::Vec2f T(tx, ty); T = scale_rot_matrix * T; // Make sure centering is correct @@ -189,31 +193,31 @@ namespace FaceAnalysis cv::warpAffine(frame, aligned_face, warp_matrix, cv::Size(out_width, out_height), cv::INTER_LINEAR); // Move the destination landmarks there as well - cv::Matx22d warp_matrix_2d(warp_matrix(0,0), warp_matrix(0,1), warp_matrix(1,0), warp_matrix(1,1)); + cv::Matx22f warp_matrix_2d(warp_matrix(0,0), warp_matrix(0,1), warp_matrix(1,0), warp_matrix(1,1)); - destination_landmarks = cv::Mat(clnf_model.detected_landmarks.reshape(1, 2).t()) * cv::Mat(warp_matrix_2d).t(); + destination_landmarks = cv::Mat(detected_landmarks.reshape(1, 2).t()) * cv::Mat(warp_matrix_2d).t(); destination_landmarks.col(0) = destination_landmarks.col(0) + warp_matrix(0,2); destination_landmarks.col(1) = destination_landmarks.col(1) + warp_matrix(1,2); // Move the eyebrows up to include more of upper face - destination_landmarks.at(0,1) -= (30/0.7)*sim_scale; - destination_landmarks.at(16,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(0,1) -= (30/0.7)*sim_scale; + destination_landmarks.at(16,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(17,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(18,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(19,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(20,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(21,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(22,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(23,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(24,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(25,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(26,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(17,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(18,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(19,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(20,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(21,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(22,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(23,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(24,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(25,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(26,1) -= (30 / 0.7)*sim_scale; destination_landmarks = cv::Mat(destination_landmarks.t()).reshape(1, 1).t(); - LandmarkDetector::PAW paw(destination_landmarks, triangulation, 0, 0, aligned_face.cols-1, aligned_face.rows-1); + PAW paw(destination_landmarks, triangulation, 0, 0, aligned_face.cols-1, aligned_face.rows-1); // Mask each of the channels (a bit of a roundabout way, but OpenCV 3.1 in debug mode doesn't seem to be able to handle a more direct way using split and merge) vector aligned_face_channels(aligned_face.channels()); @@ -355,4 +359,298 @@ namespace FaceAnalysis new_descriptor.copyTo(descriptors.row(row_to_change)); } + //=========================================================================== + // Point set and landmark manipulation functions + //=========================================================================== + // Using Kabsch's algorithm for aligning shapes + //This assumes that align_from and align_to are already mean normalised + cv::Matx22f AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to) + { + + cv::SVD svd(align_from.t() * align_to); + + // make sure no reflection is there + // corr ensures that we do only rotaitons and not reflections + float d = cv::determinant(svd.vt.t() * svd.u.t()); + + cv::Matx22f corr = cv::Matx22f::eye(); + if (d > 0) + { + corr(1, 1) = 1; + } + else + { + corr(1, 1) = -1; + } + + cv::Matx22f R; + cv::Mat(svd.vt.t()*cv::Mat(corr)*svd.u.t()).copyTo(R); + + return R; + } + + //============================================================================= + // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other + cv::Matx22f AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst) + { + int n = src.rows; + + // First we mean normalise both src and dst + float mean_src_x = cv::mean(src.col(0))[0]; + float mean_src_y = cv::mean(src.col(1))[0]; + + float mean_dst_x = cv::mean(dst.col(0))[0]; + float mean_dst_y = cv::mean(dst.col(1))[0]; + + cv::Mat_ src_mean_normed = src.clone(); + src_mean_normed.col(0) = src_mean_normed.col(0) - mean_src_x; + src_mean_normed.col(1) = src_mean_normed.col(1) - mean_src_y; + + cv::Mat_ dst_mean_normed = dst.clone(); + dst_mean_normed.col(0) = dst_mean_normed.col(0) - mean_dst_x; + dst_mean_normed.col(1) = dst_mean_normed.col(1) - mean_dst_y; + + // Find the scaling factor of each + cv::Mat src_sq; + cv::pow(src_mean_normed, 2, src_sq); + + cv::Mat dst_sq; + cv::pow(dst_mean_normed, 2, dst_sq); + + float s_src = sqrt(cv::sum(src_sq)[0] / n); + float s_dst = sqrt(cv::sum(dst_sq)[0] / n); + + src_mean_normed = src_mean_normed / s_src; + dst_mean_normed = dst_mean_normed / s_dst; + + float s = s_dst / s_src; + + // Get the rotation + cv::Matx22f R = AlignShapesKabsch2D(src_mean_normed, dst_mean_normed); + + cv::Matx22f A; + cv::Mat(s * R).copyTo(A); + + cv::Mat_ aligned = (cv::Mat(cv::Mat(A) * src.t())).t(); + cv::Mat_ offset = dst - aligned; + + float t_x = cv::mean(offset.col(0))[0]; + float t_y = cv::mean(offset.col(1))[0]; + + return A; + + } + + + //=========================================================================== + // Visualisation functions + //=========================================================================== + void Project(cv::Mat_& dest, const cv::Mat_& mesh, float fx, float fy, float cx, float cy) + { + dest = cv::Mat_(mesh.rows, 2, 0.0); + + int num_points = mesh.rows; + + float X, Y, Z; + + + cv::Mat_::const_iterator mData = mesh.begin(); + cv::Mat_::iterator projected = dest.begin(); + + for (int i = 0; i < num_points; i++) + { + // Get the points + X = *(mData++); + Y = *(mData++); + Z = *(mData++); + + float x; + float y; + + // if depth is 0 the projection is different + if (Z != 0) + { + x = ((X * fx / Z) + cx); + y = ((Y * fy / Z) + cy); + } + else + { + x = X; + y = Y; + } + + // Project and store in dest matrix + (*projected++) = x; + (*projected++) = y; + } + + } + + //=========================================================================== + // Angle representation conversion helpers + //=========================================================================== + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + cv::Matx33f Euler2RotationMatrix(const cv::Vec3f& eulerAngles) + { + cv::Matx33f rotation_matrix; + + float s1 = sin(eulerAngles[0]); + float s2 = sin(eulerAngles[1]); + float s3 = sin(eulerAngles[2]); + + float c1 = cos(eulerAngles[0]); + float c2 = cos(eulerAngles[1]); + float c3 = cos(eulerAngles[2]); + + rotation_matrix(0, 0) = c2 * c3; + rotation_matrix(0, 1) = -c2 *s3; + rotation_matrix(0, 2) = s2; + rotation_matrix(1, 0) = c1 * s3 + c3 * s1 * s2; + rotation_matrix(1, 1) = c1 * c3 - s1 * s2 * s3; + rotation_matrix(1, 2) = -c2 * s1; + rotation_matrix(2, 0) = s1 * s3 - c1 * c3 * s2; + rotation_matrix(2, 1) = c3 * s1 + c1 * s2 * s3; + rotation_matrix(2, 2) = c1 * c2; + + return rotation_matrix; + } + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + cv::Vec3f RotationMatrix2Euler(const cv::Matx33f& rotation_matrix) + { + float q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0f; + float q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0f*q0); + float q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0f*q0); + float q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0f*q0); + + float t1 = 2.0f * (q0*q2 + q1*q3); + + float yaw = asin(2.0 * (q0*q2 + q1*q3)); + float pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3); + float roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3); + + return cv::Vec3f(pitch, yaw, roll); + } + + cv::Vec3f Euler2AxisAngle(const cv::Vec3f& euler) + { + cv::Matx33f rotMatrix = Euler2RotationMatrix(euler); + cv::Vec3f axis_angle; + cv::Rodrigues(rotMatrix, axis_angle); + return axis_angle; + } + + cv::Vec3f AxisAngle2Euler(const cv::Vec3f& axis_angle) + { + cv::Matx33f rotation_matrix; + cv::Rodrigues(axis_angle, rotation_matrix); + return RotationMatrix2Euler(rotation_matrix); + } + + cv::Matx33f AxisAngle2RotationMatrix(const cv::Vec3f& axis_angle) + { + cv::Matx33f rotation_matrix; + cv::Rodrigues(axis_angle, rotation_matrix); + return rotation_matrix; + } + + cv::Vec3f RotationMatrix2AxisAngle(const cv::Matx33f& rotation_matrix) + { + cv::Vec3f axis_angle; + cv::Rodrigues(rotation_matrix, axis_angle); + return axis_angle; + } + + //============================================================================ + // Matrix reading functionality + //============================================================================ + + // Reading in a matrix from a stream + void ReadMat(std::ifstream& stream, cv::Mat &output_mat) + { + // Read in the number of rows, columns and the data type + int row, col, type; + + stream >> row >> col >> type; + + output_mat = cv::Mat(row, col, type); + + switch (output_mat.type()) + { + case CV_64FC1: + { + cv::MatIterator_ begin_it = output_mat.begin(); + cv::MatIterator_ end_it = output_mat.end(); + + while (begin_it != end_it) + { + stream >> *begin_it++; + } + } + break; + case CV_32FC1: + { + cv::MatIterator_ begin_it = output_mat.begin(); + cv::MatIterator_ end_it = output_mat.end(); + + while (begin_it != end_it) + { + stream >> *begin_it++; + } + } + break; + case CV_32SC1: + { + cv::MatIterator_ begin_it = output_mat.begin(); + cv::MatIterator_ end_it = output_mat.end(); + while (begin_it != end_it) + { + stream >> *begin_it++; + } + } + break; + case CV_8UC1: + { + cv::MatIterator_ begin_it = output_mat.begin(); + cv::MatIterator_ end_it = output_mat.end(); + while (begin_it != end_it) + { + stream >> *begin_it++; + } + } + break; + default: + printf("ERROR(%s,%d) : Unsupported Matrix type %d!\n", __FILE__, __LINE__, output_mat.type()); abort(); + + + } + } + + void ReadMatBin(std::ifstream& stream, cv::Mat &output_mat) + { + // Read in the number of rows, columns and the data type + int row, col, type; + + stream.read((char*)&row, 4); + stream.read((char*)&col, 4); + stream.read((char*)&type, 4); + + output_mat = cv::Mat(row, col, type); + int size = output_mat.rows * output_mat.cols * output_mat.elemSize(); + stream.read((char *)output_mat.data, size); + + } + + // Skipping lines that start with # (together with empty lines) + void SkipComments(std::ifstream& stream) + { + while (stream.peek() == '#' || stream.peek() == '\n' || stream.peek() == ' ' || stream.peek() == '\r') + { + std::string skipped; + std::getline(stream, skipped); + } + } + + } \ No newline at end of file diff --git a/lib/local/FaceAnalyser/src/PAW.cpp b/lib/local/FaceAnalyser/src/PAW.cpp new file mode 100644 index 0000000..542c68e --- /dev/null +++ b/lib/local/FaceAnalyser/src/PAW.cpp @@ -0,0 +1,514 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + + +#include "PAW.h" + +// OpenCV includes +#include +#include + +#include "Face_utils.h" + +using namespace FaceAnalysis; + +// Copy constructor +PAW::PAW(const PAW& other) : destination_landmarks(other.destination_landmarks.clone()), source_landmarks(other.source_landmarks.clone()), triangulation(other.triangulation.clone()), +triangle_id(other.triangle_id.clone()), pixel_mask(other.pixel_mask.clone()), coefficients(other.coefficients.clone()), alpha(other.alpha.clone()), beta(other.beta.clone()), map_x(other.map_x.clone()), map_y(other.map_y.clone()) +{ + this->number_of_pixels = other.number_of_pixels; + this->min_x = other.min_x; + this->min_y = other.min_y; +} + +// A constructor from destination shape and triangulation +PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation) +{ + // Initialise some variables directly + this->destination_landmarks = destination_shape; + this->triangulation = triangulation; + + int num_points = destination_shape.rows/2; + + int num_tris = triangulation.rows; + + // Pre-compute the rest + alpha = cv::Mat_(num_tris, 3); + beta = cv::Mat_(num_tris, 3); + + cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); + cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); + + // Create a vector representation of the control points + std::vector> destination_points; + + for (int tri = 0; tri < num_tris; ++tri) + { + int j = triangulation.at(tri, 0); + int k = triangulation.at(tri, 1); + int l = triangulation.at(tri, 2); + + float c1 = ys.at(l) - ys.at(j); + float c2 = xs.at(l) - xs.at(j); + float c4 = ys.at(k) - ys.at(j); + float c3 = xs.at(k) - xs.at(j); + + float c5 = c3*c1 - c2*c4; + + alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; + alpha.at(tri, 1) = c1/c5; + alpha.at(tri, 2) = -c2/c5; + + beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; + beta.at(tri, 1) = -c4/c5; + beta.at(tri, 2) = c3/c5; + + // Add points corresponding to triangles as optimisation + std::vector triangle_points(10); + + triangle_points[0] = xs.at(j); + triangle_points[1] = ys.at(j); + triangle_points[2] = xs.at(k); + triangle_points[3] = ys.at(k); + triangle_points[4] = xs.at(l); + triangle_points[5] = ys.at(l); + + cv::Vec3f xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); + cv::Vec3f ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); + + double min_x, max_x, min_y, max_y; + cv::minMaxIdx(xs_three, &min_x, &max_x); + cv::minMaxIdx(ys_three, &min_y, &max_y); + + triangle_points[6] = (float) max_x; + triangle_points[7] = (float) max_y; + + triangle_points[8] = (float) min_x; + triangle_points[9] = (float) min_y; + + destination_points.push_back(triangle_points); + + } + + double max_x; + double max_y; + double min_x_d; + double min_y_d; + + minMaxLoc(xs, &min_x_d, &max_x); + minMaxLoc(ys, &min_y_d, &max_y); + + min_x = min_x_d; + min_y = min_y_d; + + int w = (int)(max_x - min_x + 1.5); + int h = (int)(max_y - min_y + 1.5); + + // Round the min_x and min_y for simplicity? + + pixel_mask = cv::Mat_(h, w, (uchar)0); + triangle_id = cv::Mat_(h, w, -1); + + int curr_tri = -1; + + for(int y = 0; y < pixel_mask.rows; y++) + { + for(int x = 0; x < pixel_mask.cols; x++) + { + curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); + // If there is a triangle at this location + if(curr_tri != -1) + { + triangle_id.at(y, x) = curr_tri; + pixel_mask.at(y, x) = 1; + } + } + } + + // Preallocate maps and coefficients + coefficients.create(num_tris, 6); + map_x.create(pixel_mask.rows,pixel_mask.cols); + map_y.create(pixel_mask.rows,pixel_mask.cols); + + +} + +// Manually define min and max values +PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, float in_min_x, float in_min_y, float in_max_x, float in_max_y) +{ + // Initialise some variables directly + this->destination_landmarks = destination_shape; + this->triangulation = triangulation; + + int num_points = destination_shape.rows/2; + + int num_tris = triangulation.rows; + + // Pre-compute the rest + alpha = cv::Mat_(num_tris, 3); + beta = cv::Mat_(num_tris, 3); + + cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); + cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); + + // Create a vector representation of the control points + std::vector> destination_points; + + for (int tri = 0; tri < num_tris; ++tri) + { + int j = triangulation.at(tri, 0); + int k = triangulation.at(tri, 1); + int l = triangulation.at(tri, 2); + + float c1 = ys.at(l) - ys.at(j); + float c2 = xs.at(l) - xs.at(j); + float c4 = ys.at(k) - ys.at(j); + float c3 = xs.at(k) - xs.at(j); + + float c5 = c3*c1 - c2*c4; + + alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; + alpha.at(tri, 1) = c1/c5; + alpha.at(tri, 2) = -c2/c5; + + beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; + beta.at(tri, 1) = -c4/c5; + beta.at(tri, 2) = c3/c5; + + // Add points corresponding to triangles as optimisation + std::vector triangle_points(10); + + triangle_points[0] = xs.at(j); + triangle_points[1] = ys.at(j); + triangle_points[2] = xs.at(k); + triangle_points[3] = ys.at(k); + triangle_points[4] = xs.at(l); + triangle_points[5] = ys.at(l); + + cv::Vec3f xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); + cv::Vec3f ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); + + double min_x, max_x, min_y, max_y; + cv::minMaxIdx(xs_three, &min_x, &max_x); + cv::minMaxIdx(ys_three, &min_y, &max_y); + + triangle_points[6] = (float)max_x; + triangle_points[7] = (float)max_y; + + triangle_points[8] = (float)min_x; + triangle_points[9] = (float)min_y; + + destination_points.push_back(triangle_points); + + } + + float max_x; + float max_y; + + min_x = in_min_x; + min_y = in_min_y; + + max_x = in_max_x; + max_y = in_max_y; + + int w = (int)(max_x - min_x + 1.5); + int h = (int)(max_y - min_y + 1.5); + + // Round the min_x and min_y for simplicity? + + pixel_mask = cv::Mat_(h, w, (uchar)0); + triangle_id = cv::Mat_(h, w, -1); + + int curr_tri = -1; + + for(int y = 0; y < pixel_mask.rows; y++) + { + for(int x = 0; x < pixel_mask.cols; x++) + { + curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); + // If there is a triangle at this location + if(curr_tri != -1) + { + triangle_id.at(y, x) = curr_tri; + pixel_mask.at(y, x) = 1; + } + } + } + + // Preallocate maps and coefficients + coefficients.create(num_tris, 6); + map_x.create(pixel_mask.rows,pixel_mask.cols); + map_y.create(pixel_mask.rows,pixel_mask.cols); + +} + +//=========================================================================== +void PAW::Read(std::ifstream& stream) +{ + + stream.read ((char*)&number_of_pixels, 4); + double min_x_d, min_y_d; + stream.read ((char*)&min_x_d, 8); + stream.read ((char*)&min_y_d, 8); + min_x = (float)min_x_d; + min_y = (float)min_y_d; + + cv::Mat_ destination_landmarks_d; + ReadMatBin(stream, destination_landmarks_d); + destination_landmarks_d.convertTo(destination_landmarks, CV_32F); + + ReadMatBin(stream, triangulation); + + ReadMatBin(stream, triangle_id); + + cv::Mat tmpMask; + ReadMatBin(stream, tmpMask); + tmpMask.convertTo(pixel_mask, CV_8U); + + cv::Mat_ alpha_d; + ReadMatBin(stream, alpha_d); + alpha_d.convertTo(alpha, CV_32F); + + cv::Mat_ beta_d; + ReadMatBin(stream, beta_d); + beta_d.convertTo(beta, CV_32F); + + map_x.create(pixel_mask.rows,pixel_mask.cols); + map_y.create(pixel_mask.rows,pixel_mask.cols); + + coefficients.create(this->NumberOfTriangles(),6); + + source_landmarks = destination_landmarks; +} + +//============================================================================= +// cropping from the source image to the destination image using the shape in s, used to determine if shape fitting converged successfully +void PAW::Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp) +{ + + // set the current shape + source_landmarks = landmarks_to_warp.clone(); + + // prepare the mapping coefficients using the current shape + this->CalcCoeff(); + + // Do the actual mapping computation (where to warp from) + this->WarpRegion(map_x, map_y); + + // Do the actual warp (with bi-linear interpolation) + remap(image_to_warp, destination_image, map_x, map_y, CV_INTER_LINEAR); + +} + + +//============================================================================= +// Calculate the warping coefficients +void PAW::CalcCoeff() +{ + int p = this->NumberOfLandmarks(); + + for(int l = 0; l < this->NumberOfTriangles(); l++) + { + + int i = triangulation.at(l,0); + int j = triangulation.at(l,1); + int k = triangulation.at(l,2); + + float c1 = source_landmarks.at(i , 0); + float c2 = source_landmarks.at(j , 0) - c1; + float c3 = source_landmarks.at(k , 0) - c1; + float c4 = source_landmarks.at(i + p, 0); + float c5 = source_landmarks.at(j + p, 0) - c4; + float c6 = source_landmarks.at(k + p, 0) - c4; + + // Get a pointer to the coefficient we will be precomputing + float *coeff = coefficients.ptr(l); + + // Extract the relevant alphas and betas + float *c_alpha = alpha.ptr(l); + float *c_beta = beta.ptr(l); + + coeff[0] = c1 + c2 * c_alpha[0] + c3 * c_beta[0]; + coeff[1] = c2 * c_alpha[1] + c3 * c_beta[1]; + coeff[2] = c2 * c_alpha[2] + c3 * c_beta[2]; + coeff[3] = c4 + c5 * c_alpha[0] + c6 * c_beta[0]; + coeff[4] = c5 * c_alpha[1] + c6 * c_beta[1]; + coeff[5] = c5 * c_alpha[2] + c6 * c_beta[2]; + } +} + +//====================================================================== +// Compute the mapping coefficients +void PAW::WarpRegion(cv::Mat_& mapx, cv::Mat_& mapy) +{ + + cv::MatIterator_ xp = mapx.begin(); + cv::MatIterator_ yp = mapy.begin(); + cv::MatIterator_ mp = pixel_mask.begin(); + cv::MatIterator_ tp = triangle_id.begin(); + + // The coefficients corresponding to the current triangle + float * a; + + // Current triangle being processed + int k=-1; + + for(int y = 0; y < pixel_mask.rows; y++) + { + float yi = float(y) + min_y; + + for(int x = 0; x < pixel_mask.cols; x++) + { + float xi = float(x) + min_x; + + if(*mp == 0) + { + *xp = -1; + *yp = -1; + } + else + { + // triangle corresponding to the current pixel + int j = *tp; + + // If it is different from the previous triangle point to new coefficients + // This will always be the case in the first iteration, hence a will not point to nothing + if(j != k) + { + // Update the coefficient pointer if a new triangle is being processed + a = coefficients.ptr(j); + k = j; + } + + //ap is now the pointer to the coefficients + float *ap = a; + + //look at the first coefficient (and increment). first coefficient is an x offset + float xo = *ap++; + //second coefficient is an x scale as a function of x + xo += *ap++ * xi; + //third coefficient ap(2) is an x scale as a function of y + *xp = float(xo + *ap++ * yi); + + //then fourth coefficient ap(3) is a y offset + float yo = *ap++; + //fifth coeff adds coeff[4]*x to y + yo += *ap++ * xi; + //final coeff adds coeff[5]*y to y + *yp = float(yo + *ap++ * yi); + + } + mp++; tp++; xp++; yp++; + } + } +} + +// ============================================================ +// Helper functions to determine which point a triangle lies in +// ============================================================ + +// Is the point (x0,y0) on same side as a half-plane defined by (x1,y1), (x2, y2), and (x3, y3) +bool PAW::sameSide(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3) +{ + + float x = (x3-x2)*(y0-y2) - (x0-x2)*(y3-y2); + float y = (x3-x2)*(y1-y2) - (x1-x2)*(y3-y2); + + return x*y >= 0; + +} + +// if point (x0, y0) is on same side for all three half-planes it is in a triangle +bool PAW::pointInTriangle(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3) +{ + bool same_1 = sameSide(x0, y0, x1, y1, x2, y2, x3, y3); + bool same_2 = sameSide(x0, y0, x2, y2, x1, y1, x3, y3); + bool same_3 = sameSide(x0, y0, x3, y3, x1, y1, x2, y2); + + return same_1 && same_2 && same_3; + +} + +// Find if a given point lies in the triangles +int PAW::findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess) +{ + + int num_tris = control_points.size(); + + int tri = -1; + + float x0 = point.x; + float y0 = point.y; + + // Allow a guess for speed (so as not to go through all triangles) + if(guess != -1) + { + + bool in_triangle = pointInTriangle(x0, y0, control_points[guess][0], control_points[guess][1], control_points[guess][2], control_points[guess][3], control_points[guess][4], control_points[guess][5]); + if(in_triangle) + { + return guess; + } + } + + + for (int i = 0; i < num_tris; ++i) + { + + float max_x = control_points[i][6]; + float max_y = control_points[i][7]; + + float min_x = control_points[i][8]; + float min_y = control_points[i][9]; + + // Skip the check if the point is outside the bounding box of the triangle + + if( max_x < x0 || min_x > x0 || max_y < y0 || min_y > y0) + { + continue; + } + + bool in_triangle = pointInTriangle(x0, y0, + control_points[i][0], control_points[i][1], + control_points[i][2], control_points[i][3], + control_points[i][4], control_points[i][5]); + + if(in_triangle) + { + tri = i; + break; + } + } + return tri; +} \ No newline at end of file diff --git a/lib/local/FaceAnalyser/src/PDM.cpp b/lib/local/FaceAnalyser/src/PDM.cpp new file mode 100644 index 0000000..7fc6c20 --- /dev/null +++ b/lib/local/FaceAnalyser/src/PDM.cpp @@ -0,0 +1,613 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + + +#include + +// OpenCV include +#include +#include + +// Math includes +#define _USE_MATH_DEFINES +#include + +#ifndef M_PI + #define M_PI 3.14159265358979323846 +#endif + +#include + +// OpenBLAS +#include +#include + +using namespace FaceAnalysis; +//=========================================================================== + +//============================================================================= +// Orthonormalising the 3x3 rotation matrix +void PDM::Orthonormalise(cv::Matx33f &R) +{ + + cv::SVD svd(R,cv::SVD::MODIFY_A); + + // get the orthogonal matrix from the initial rotation matrix + cv::Mat_ X = svd.u*svd.vt; + + // This makes sure that the handedness is preserved and no reflection happened + // by making sure the determinant is 1 and not -1 + cv::Mat_ W = cv::Mat_::eye(3,3); + float d = determinant(X); + W(2,2) = determinant(X); + cv::Mat Rt = svd.u*W*svd.vt; + + Rt.copyTo(R); + +} + +// A copy constructor +PDM::PDM(const PDM& other) { + + // Make sure the matrices are allocated properly + this->mean_shape = other.mean_shape.clone(); + this->princ_comp = other.princ_comp.clone(); + this->eigen_values = other.eigen_values.clone(); +} + +//=========================================================================== +// Compute the 3D representation of shape (in object space) using the local parameters +void PDM::CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& p_local) const +{ + out_shape.create(mean_shape.rows, mean_shape.cols); + out_shape = mean_shape + princ_comp * p_local; +} + +//=========================================================================== +// Get the 2D shape (in image space) from global and local parameters +void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6f& params_global) const +{ + + int n = this->NumberOfPoints(); + + float s = params_global[0]; // scaling factor + float tx = params_global[4]; // x offset + float ty = params_global[5]; // y offset + + // get the rotation matrix from the euler angles + cv::Vec3f euler(params_global[1], params_global[2], params_global[3]); + cv::Matx33f currRot = Euler2RotationMatrix(euler); + + // get the 3D shape of the object + cv::Mat_ Shape_3D = mean_shape + princ_comp * params_local; + + // create the 2D shape matrix (if it has not been defined yet) + if ((out_shape.rows != mean_shape.rows) || (out_shape.cols != 1)) + { + out_shape.create(2 * n, 1); + } + // for every vertex + for (int i = 0; i < n; i++) + { + // Transform this using the weak-perspective mapping to 2D from 3D + out_shape.at(i, 0) = s * (currRot(0, 0) * Shape_3D.at(i, 0) + currRot(0, 1) * Shape_3D.at(i + n, 0) + currRot(0, 2) * Shape_3D.at(i + n * 2, 0)) + tx; + out_shape.at(i + n, 0) = s * (currRot(1, 0) * Shape_3D.at(i, 0) + currRot(1, 1) * Shape_3D.at(i + n, 0) + currRot(1, 2) * Shape_3D.at(i + n * 2, 0)) + ty; + } +} + +//=========================================================================== +// provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box +// This all assumes that the bounding box describes face from left outline to right outline of the face and chin to eyebrows +void PDM::CalcParams(cv::Vec6f& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3f rotation) const +{ + + // get the shape instance based on local params + cv::Mat_ current_shape(mean_shape.size()); + + CalcShape3D(current_shape, params_local); + + // rotate the shape + cv::Matx33f rotation_matrix = Euler2RotationMatrix(rotation); + + cv::Mat_ reshaped = current_shape.reshape(1, 3); + + cv::Mat rotated_shape = (cv::Mat(rotation_matrix) * reshaped); + + // Get the width of expected shape + double min_x; + double max_x; + cv::minMaxLoc(rotated_shape.row(0), &min_x, &max_x); + + double min_y; + double max_y; + cv::minMaxLoc(rotated_shape.row(1), &min_y, &max_y); + + float width = (float)abs(min_x - max_x); + float height = (float)abs(min_y - max_y); + + float scaling = ((bounding_box.width / width) + (bounding_box.height / height)) / 2.0f; + + // The estimate of face center also needs some correction + float tx = bounding_box.x + bounding_box.width / 2; + float ty = bounding_box.y + bounding_box.height / 2; + + // Correct it so that the bounding box is just around the minimum and maximum point in the initialised face + tx = tx - scaling * (min_x + max_x) / 2.0f; + ty = ty - scaling * (min_y + max_y) / 2.0f; + + out_params_global = cv::Vec6f(scaling, rotation[0], rotation[1], rotation[2], tx, ty); +} + +//=========================================================================== +// provided the model parameters, compute the bounding box of a face +// The bounding box describes face from left outline to right outline of the face and chin to eyebrows +void PDM::CalcBoundingBox(cv::Rect_& out_bounding_box, const cv::Vec6f& params_global, const cv::Mat_& params_local) const +{ + + // get the shape instance based on local params + cv::Mat_ current_shape; + CalcShape2D(current_shape, params_local, params_global); + + // Get the width of expected shape + double min_x; + double max_x; + cv::minMaxLoc(current_shape(cv::Rect(0, 0, 1, this->NumberOfPoints())), &min_x, &max_x); + + double min_y; + double max_y; + cv::minMaxLoc(current_shape(cv::Rect(0, this->NumberOfPoints(), 1, this->NumberOfPoints())), &min_y, &max_y); + + float width = (float)abs(min_x - max_x); + float height = (float)abs(min_y - max_y); + + out_bounding_box = cv::Rect_(min_x, min_y, width, height); +} + +//=========================================================================== +// Calculate the PDM's Jacobian over rigid parameters (rotation, translation and scaling), the additional input W represents trust for each of the landmarks and is part of Non-Uniform RLMS +void PDM::ComputeRigidJacobian(const cv::Mat_& p_local, const cv::Vec6f& params_global, cv::Mat_ &Jacob, const cv::Mat_ W, cv::Mat_ &Jacob_t_w) const +{ + + // number of verts + int n = this->NumberOfPoints(); + + Jacob.create(n * 2, 6); + + float X, Y, Z; + + float s = params_global[0]; + + cv::Mat_ shape_3D; + this->CalcShape3D(shape_3D, p_local); + + // Get the rotation matrix + cv::Vec3f euler(params_global[1], params_global[2], params_global[3]); + cv::Matx33f currRot = Euler2RotationMatrix(euler); + + float r11 = currRot(0, 0); + float r12 = currRot(0, 1); + float r13 = currRot(0, 2); + float r21 = currRot(1, 0); + float r22 = currRot(1, 1); + float r23 = currRot(1, 2); + float r31 = currRot(2, 0); + float r32 = currRot(2, 1); + float r33 = currRot(2, 2); + + cv::MatIterator_ Jx = Jacob.begin(); + cv::MatIterator_ Jy = Jx + n * 6; + + for (int i = 0; i < n; i++) + { + + X = shape_3D.at(i, 0); + Y = shape_3D.at(i + n, 0); + Z = shape_3D.at(i + n * 2, 0); + + // The rigid jacobian from the axis angle rotation matrix approximation using small angle assumption (R * R') + // where R' = [1, -wz, wy + // wz, 1, -wx + // -wy, wx, 1] + // And this is derived using the small angle assumption on the axis angle rotation matrix parametrisation + + // scaling term + *Jx++ = (X * r11 + Y * r12 + Z * r13); + *Jy++ = (X * r21 + Y * r22 + Z * r23); + + // rotation terms + *Jx++ = (s * (Y * r13 - Z * r12)); + *Jy++ = (s * (Y * r23 - Z * r22)); + *Jx++ = (-s * (X * r13 - Z * r11)); + *Jy++ = (-s * (X * r23 - Z * r21)); + *Jx++ = (s * (X * r12 - Y * r11)); + *Jy++ = (s * (X * r22 - Y * r21)); + + // translation terms + *Jx++ = 1.0f; + *Jy++ = 0.0f; + *Jx++ = 0.0f; + *Jy++ = 1.0f; + + } + + cv::Mat Jacob_w = cv::Mat::zeros(Jacob.rows, Jacob.cols, Jacob.type()); + + Jx = Jacob.begin(); + Jy = Jx + n * 6; + + cv::MatIterator_ Jx_w = Jacob_w.begin(); + cv::MatIterator_ Jy_w = Jx_w + n * 6; + + // Iterate over all Jacobian values and multiply them by the weight in diagonal of W + for (int i = 0; i < n; i++) + { + float w_x = W.at(i, i); + float w_y = W.at(i + n, i + n); + + for (int j = 0; j < Jacob.cols; ++j) + { + *Jx_w++ = *Jx++ * w_x; + *Jy_w++ = *Jy++ * w_y; + } + } + + Jacob_t_w = Jacob_w.t(); +} + +//=========================================================================== +// Calculate the PDM's Jacobian over all parameters (rigid and non-rigid), the additional input W represents trust for each of the landmarks and is part of Non-Uniform RLMS +void PDM::ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6f& params_global, cv::Mat_ &Jacobian, const cv::Mat_ W, cv::Mat_ &Jacob_t_w) const +{ + + // number of vertices + int n = this->NumberOfPoints(); + + // number of non-rigid parameters + int m = this->NumberOfModes(); + + Jacobian.create(n * 2, 6 + m); + + float X, Y, Z; + + float s = params_global[0]; + + cv::Mat_ shape_3D; + this->CalcShape3D(shape_3D, params_local); + + cv::Vec3f euler(params_global[1], params_global[2], params_global[3]); + cv::Matx33f currRot = Euler2RotationMatrix(euler); + + float r11 = currRot(0, 0); + float r12 = currRot(0, 1); + float r13 = currRot(0, 2); + float r21 = currRot(1, 0); + float r22 = currRot(1, 1); + float r23 = currRot(1, 2); + float r31 = currRot(2, 0); + float r32 = currRot(2, 1); + float r33 = currRot(2, 2); + + cv::MatIterator_ Jx = Jacobian.begin(); + cv::MatIterator_ Jy = Jx + n * (6 + m); + cv::MatConstIterator_ Vx = this->princ_comp.begin(); + cv::MatConstIterator_ Vy = Vx + n*m; + cv::MatConstIterator_ Vz = Vy + n*m; + + for (int i = 0; i < n; i++) + { + + X = shape_3D.at(i, 0); + Y = shape_3D.at(i + n, 0); + Z = shape_3D.at(i + n * 2, 0); + + // The rigid jacobian from the axis angle rotation matrix approximation using small angle assumption (R * R') + // where R' = [1, -wz, wy + // wz, 1, -wx + // -wy, wx, 1] + // And this is derived using the small angle assumption on the axis angle rotation matrix parametrisation + + // scaling term + *Jx++ = (X * r11 + Y * r12 + Z * r13); + *Jy++ = (X * r21 + Y * r22 + Z * r23); + + // rotation terms + *Jx++ = (s * (Y * r13 - Z * r12)); + *Jy++ = (s * (Y * r23 - Z * r22)); + *Jx++ = (-s * (X * r13 - Z * r11)); + *Jy++ = (-s * (X * r23 - Z * r21)); + *Jx++ = (s * (X * r12 - Y * r11)); + *Jy++ = (s * (X * r22 - Y * r21)); + + // translation terms + *Jx++ = 1.0f; + *Jy++ = 0.0f; + *Jx++ = 0.0f; + *Jy++ = 1.0f; + + for (int j = 0; j < m; j++, ++Vx, ++Vy, ++Vz) + { + // How much the change of the non-rigid parameters (when object is rotated) affect 2D motion + *Jx++ = (s*(r11*(*Vx) + r12*(*Vy) + r13*(*Vz))); + *Jy++ = (s*(r21*(*Vx) + r22*(*Vy) + r23*(*Vz))); + } + } + + // Adding the weights here + cv::Mat Jacob_w = Jacobian.clone(); + + if (cv::trace(W)[0] != W.rows) + { + Jx = Jacobian.begin(); + Jy = Jx + n*(6 + m); + + cv::MatIterator_ Jx_w = Jacob_w.begin(); + cv::MatIterator_ Jy_w = Jx_w + n*(6 + m); + + // Iterate over all Jacobian values and multiply them by the weight in diagonal of W + for (int i = 0; i < n; i++) + { + float w_x = W.at(i, i); + float w_y = W.at(i + n, i + n); + + for (int j = 0; j < Jacobian.cols; ++j) + { + *Jx_w++ = *Jx++ * w_x; + *Jy_w++ = *Jy++ * w_y; + } + } + } + Jacob_t_w = Jacob_w.t(); + +} + + +//=========================================================================== +// Updating the parameters (more details in my thesis) +void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& params_local, cv::Vec6f& params_global) const +{ + + // The scaling and translation parameters can be just added + params_global[0] += delta_p.at(0, 0); + params_global[4] += delta_p.at(4, 0); + params_global[5] += delta_p.at(5, 0); + + // get the original rotation matrix + cv::Vec3f eulerGlobal(params_global[1], params_global[2], params_global[3]); + cv::Matx33f R1 = Euler2RotationMatrix(eulerGlobal); + + // construct R' = [1, -wz, wy + // wz, 1, -wx + // -wy, wx, 1] + cv::Matx33f R2 = cv::Matx33f::eye(); + + R2(1, 2) = -1.0*(R2(2, 1) = delta_p.at(1, 0)); + R2(2, 0) = -1.0*(R2(0, 2) = delta_p.at(2, 0)); + R2(0, 1) = -1.0*(R2(1, 0) = delta_p.at(3, 0)); + + // Make sure it's orthonormal + Orthonormalise(R2); + + // Combine rotations + cv::Matx33f R3 = R1 *R2; + + // Extract euler angle (through axis angle first to make sure it's legal) + cv::Vec3f axis_angle = RotationMatrix2AxisAngle(R3); + cv::Vec3f euler = AxisAngle2Euler(axis_angle); + + params_global[1] = euler[0]; + params_global[2] = euler[1]; + params_global[3] = euler[2]; + + // Local parameter update, just simple addition + if (delta_p.rows > 6) + { + params_local = params_local + delta_p(cv::Rect(0, 6, 1, this->NumberOfModes())); + } + +} +// void CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation = cv::Vec3d(0.0)) const; +void PDM::CalcParams(cv::Vec6f& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3f rotation) const +{ + + int m = this->NumberOfModes(); + int n = this->NumberOfPoints(); + + // The new number of points + n = this->mean_shape.rows / 3; + + // Compute the initial global parameters + double min_x; + double max_x; + cv::minMaxLoc(landmark_locations(cv::Rect(0, 0, 1, this->NumberOfPoints())), &min_x, &max_x); + + double min_y; + double max_y; + cv::minMaxLoc(landmark_locations(cv::Rect(0, this->NumberOfPoints(), 1, this->NumberOfPoints())), &min_y, &max_y); + + float width = (float)abs(min_x - max_x); + float height = (float)abs(min_y - max_y); + + cv::Rect_ model_bbox; + CalcBoundingBox(model_bbox, cv::Vec6d(1.0, 0.0, 0.0, 0.0, 0.0, 0.0), cv::Mat_(this->NumberOfModes(), 1, 0.0)); + + cv::Rect bbox((int)min_x, (int)min_y, (int)width, (int)height); + + float scaling = ((width / model_bbox.width) + (height / model_bbox.height)) / 2; + + cv::Vec3f rotation_init(rotation[0], rotation[1], rotation[2]); + cv::Matx33f R = Euler2RotationMatrix(rotation_init); + cv::Vec2f translation((min_x + max_x) / 2.0, (min_y + max_y) / 2.0); + + cv::Mat_ loc_params(this->NumberOfModes(),1, 0.0); + cv::Vec6f glob_params(scaling, rotation_init[0], rotation_init[1], rotation_init[2], translation[0], translation[1]); + + // get the 3D shape of the object + cv::Mat_ shape_3D = mean_shape + princ_comp * loc_params; + + cv::Mat_ curr_shape(2*n, 1); + + // for every vertex + for(int i = 0; i < n; i++) + { + // Transform this using the weak-perspective mapping to 2D from 3D + curr_shape.at(i ,0) = scaling * ( R(0,0) * shape_3D.at(i, 0) + R(0,1) * shape_3D.at(i+n ,0) + R(0,2) * shape_3D.at(i+n*2,0) ) + translation[0]; + curr_shape.at(i+n,0) = scaling * ( R(1,0) * shape_3D.at(i, 0) + R(1,1) * shape_3D.at(i+n ,0) + R(1,2) * shape_3D.at(i+n*2,0) ) + translation[1]; + } + + float currError = cv::norm(curr_shape - landmark_locations); + + cv::Mat_ regularisations = cv::Mat_::zeros(1, 6 + m); + + float reg_factor = 1; + + // Setting the regularisation to the inverse of eigenvalues + cv::Mat(reg_factor / this->eigen_values).copyTo(regularisations(cv::Rect(6, 0, m, 1))); + regularisations = cv::Mat::diag(regularisations.t()); + + cv::Mat_ WeightMatrix = cv::Mat_::eye(n*2, n*2); + + int not_improved_in = 0; + + for (size_t i = 0; i < 1000; ++i) + { + // get the 3D shape of the object + shape_3D = mean_shape + princ_comp * loc_params; + + shape_3D = shape_3D.reshape(1, 3); + + cv::Matx23f R_2D(R(0,0), R(0,1), R(0,2), R(1,0), R(1,1), R(1,2)); + + cv::Mat_ curr_shape_2D = scaling * shape_3D.t() * cv::Mat(R_2D).t(); + curr_shape_2D.col(0) = curr_shape_2D.col(0) + translation(0); + curr_shape_2D.col(1) = curr_shape_2D.col(1) + translation(1); + + curr_shape_2D = cv::Mat(curr_shape_2D.t()).reshape(1, n * 2); + + cv::Mat_ error_resid; + cv::Mat(landmark_locations - curr_shape_2D).convertTo(error_resid, CV_32F); + + cv::Mat_ J, J_w_t; + this->ComputeJacobian(loc_params, glob_params, J, WeightMatrix, J_w_t); + + // projection of the meanshifts onto the jacobians (using the weighted Jacobian, see Baltrusaitis 2013) + cv::Mat_ J_w_t_m = J_w_t * error_resid; + + // Add the regularisation term + J_w_t_m(cv::Rect(0,6,1, m)) = J_w_t_m(cv::Rect(0,6,1, m)) - regularisations(cv::Rect(6,6, m, m)) * loc_params; + + cv::Mat_ Hessian = regularisations.clone(); + + // Perform matrix multiplication in OpenBLAS (fortran call) + float alpha1 = 1.0; + float beta1 = 1.0; + sgemm_("N", "N", &J.cols, &J_w_t.rows, &J_w_t.cols, &alpha1, (float*)J.data, &J.cols, (float*)J_w_t.data, &J_w_t.cols, &beta1, (float*)Hessian.data, &J.cols); + + // Above is a fast (but ugly) version of + // cv::Mat_ Hessian2 = J_w_t * J + regularisations; + + // Solve for the parameter update (from Baltrusaitis 2013 based on eq (36) Saragih 2011) + cv::Mat_ param_update; + cv::solve(Hessian, J_w_t_m, param_update, CV_CHOLESKY); + + // To not overshoot, have the gradient decent rate a bit smaller + param_update = 0.75 * param_update; + + UpdateModelParameters(param_update, loc_params, glob_params); + + scaling = glob_params[0]; + rotation_init[0] = glob_params[1]; + rotation_init[1] = glob_params[2]; + rotation_init[2] = glob_params[3]; + + translation[0] = glob_params[4]; + translation[1] = glob_params[5]; + + R = Euler2RotationMatrix(rotation_init); + + R_2D(0,0) = R(0,0);R_2D(0,1) = R(0,1); R_2D(0,2) = R(0,2); + R_2D(1,0) = R(1,0);R_2D(1,1) = R(1,1); R_2D(1,2) = R(1,2); + + curr_shape_2D = scaling * shape_3D.t() * cv::Mat(R_2D).t(); + curr_shape_2D.col(0) = curr_shape_2D.col(0) + translation(0); + curr_shape_2D.col(1) = curr_shape_2D.col(1) + translation(1); + + curr_shape_2D = cv::Mat(curr_shape_2D.t()).reshape(1, n * 2); + + float error = cv::norm(curr_shape_2D - landmark_locations); + + if(0.999 * currError < error) + { + not_improved_in++; + if (not_improved_in == 3) + { + break; + } + } + + currError = error; + + } + + out_params_global = glob_params; + out_params_local = loc_params; + +} + +void PDM::Read(std::string location) +{ + + std::ifstream pdmLoc(location, std::ios_base::in); + + SkipComments(pdmLoc); + + // Reading mean values + cv::Mat_ mean_shape_d; + ReadMat(pdmLoc, mean_shape_d); + mean_shape_d.convertTo(mean_shape, CV_32F); // Moving things to floats for speed + + SkipComments(pdmLoc); + + // Reading principal components + cv::Mat_ princ_comp_d; + ReadMat(pdmLoc, princ_comp_d); + princ_comp_d.convertTo(princ_comp, CV_32F); + + SkipComments(pdmLoc); + + // Reading eigenvalues + cv::Mat_ eigen_values_d; + ReadMat(pdmLoc, eigen_values_d); + eigen_values_d.convertTo(eigen_values, CV_32F); + +} diff --git a/lib/local/FaceAnalyser/src/SVM_dynamic_lin.cpp b/lib/local/FaceAnalyser/src/SVM_dynamic_lin.cpp index edf4ac4..9c224c0 100644 --- a/lib/local/FaceAnalyser/src/SVM_dynamic_lin.cpp +++ b/lib/local/FaceAnalyser/src/SVM_dynamic_lin.cpp @@ -31,11 +31,10 @@ // in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. // /////////////////////////////////////////////////////////////////////////////// +#include "Face_utils.h" #include "SVM_dynamic_lin.h" -#include "LandmarkCoreIncludes.h" - using namespace FaceAnalysis; void SVM_dynamic_lin::Read(std::ifstream& stream, const std::vector& au_names) @@ -43,20 +42,20 @@ void SVM_dynamic_lin::Read(std::ifstream& stream, const std::vector if(this->means.empty()) { - LandmarkDetector::ReadMatBin(stream, this->means); + ReadMatBin(stream, this->means); } else { cv::Mat_ m_tmp; - LandmarkDetector::ReadMatBin(stream, m_tmp); + ReadMatBin(stream, m_tmp); if(cv::norm(m_tmp - this->means > 0.00001)) { - cout << "Something went wrong with the SVM dynamic classifiers" << endl; + std::cout << "Something went wrong with the SVM dynamic classifiers" << std::endl; } } cv::Mat_ support_vectors_curr; - LandmarkDetector::ReadMatBin(stream, support_vectors_curr); + ReadMatBin(stream, support_vectors_curr); double bias; stream.read((char *)&bias, 8); diff --git a/lib/local/FaceAnalyser/src/SVM_static_lin.cpp b/lib/local/FaceAnalyser/src/SVM_static_lin.cpp index d125dd1..89b04ad 100644 --- a/lib/local/FaceAnalyser/src/SVM_static_lin.cpp +++ b/lib/local/FaceAnalyser/src/SVM_static_lin.cpp @@ -31,11 +31,10 @@ // in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. // /////////////////////////////////////////////////////////////////////////////// +#include "Face_utils.h" #include "SVM_static_lin.h" -#include "LandmarkCoreIncludes.h" - using namespace FaceAnalysis; void SVM_static_lin::Read(std::ifstream& stream, const std::vector& au_names) @@ -43,20 +42,20 @@ void SVM_static_lin::Read(std::ifstream& stream, const std::vector& if(this->means.empty()) { - LandmarkDetector::ReadMatBin(stream, this->means); + ReadMatBin(stream, this->means); } else { cv::Mat_ m_tmp; - LandmarkDetector::ReadMatBin(stream, m_tmp); + ReadMatBin(stream, m_tmp); if(cv::norm(m_tmp - this->means > 0.00001)) { - cout << "Something went wrong with the SVM static classifiers" << endl; + std::cout << "Something went wrong with the SVM static classifiers" << std::endl; } } cv::Mat_ support_vectors_curr; - LandmarkDetector::ReadMatBin(stream, support_vectors_curr); + ReadMatBin(stream, support_vectors_curr); double bias; stream.read((char *)&bias, 8); diff --git a/lib/local/FaceAnalyser/src/SVR_dynamic_lin_regressors.cpp b/lib/local/FaceAnalyser/src/SVR_dynamic_lin_regressors.cpp index 5f9cd71..1255b3d 100644 --- a/lib/local/FaceAnalyser/src/SVR_dynamic_lin_regressors.cpp +++ b/lib/local/FaceAnalyser/src/SVR_dynamic_lin_regressors.cpp @@ -32,9 +32,9 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "SVR_dynamic_lin_regressors.h" +#include "Face_utils.h" -#include "LandmarkCoreIncludes.h" +#include "SVR_dynamic_lin_regressors.h" using namespace FaceAnalysis; @@ -49,20 +49,20 @@ void SVR_dynamic_lin_regressors::Read(std::ifstream& stream, const std::vectormeans.empty()) { - LandmarkDetector::ReadMatBin(stream, this->means); + ReadMatBin(stream, this->means); } else { cv::Mat_ m_tmp; - LandmarkDetector::ReadMatBin(stream, m_tmp); + ReadMatBin(stream, m_tmp); if(cv::norm(m_tmp - this->means > 0.00001)) { - cout << "Something went wrong with the SVR dynamic regressors" << endl; + std::cout << "Something went wrong with the SVR dynamic regressors" << std::endl; } } cv::Mat_ support_vectors_curr; - LandmarkDetector::ReadMatBin(stream, support_vectors_curr); + ReadMatBin(stream, support_vectors_curr); double bias; stream.read((char *)&bias, 8); diff --git a/lib/local/FaceAnalyser/src/SVR_static_lin_regressors.cpp b/lib/local/FaceAnalyser/src/SVR_static_lin_regressors.cpp index 697d3a7..71e26b7 100644 --- a/lib/local/FaceAnalyser/src/SVR_static_lin_regressors.cpp +++ b/lib/local/FaceAnalyser/src/SVR_static_lin_regressors.cpp @@ -32,9 +32,9 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "SVR_static_lin_regressors.h" +#include "Face_utils.h" -#include "LandmarkCoreIncludes.h" +#include "SVR_static_lin_regressors.h" using namespace FaceAnalysis; @@ -43,20 +43,20 @@ void SVR_static_lin_regressors::Read(std::ifstream& stream, const std::vectormeans.empty()) { - LandmarkDetector::ReadMatBin(stream, this->means); + ReadMatBin(stream, this->means); } else { cv::Mat_ m_tmp; - LandmarkDetector::ReadMatBin(stream, m_tmp); + ReadMatBin(stream, m_tmp); if(cv::norm(m_tmp - this->means > 0.00001)) { - cout << "Something went wrong with the SVR static regressors" << endl; + std::cout << "Something went wrong with the SVR static regressors" << std::endl; } } cv::Mat_ support_vectors_curr; - LandmarkDetector::ReadMatBin(stream, support_vectors_curr); + ReadMatBin(stream, support_vectors_curr); double bias; stream.read((char *)&bias, 8); diff --git a/matlab_version/face_detection/mtcnn/setup.m b/matlab_version/face_detection/mtcnn/setup.m new file mode 100644 index 0000000..3d4150f --- /dev/null +++ b/matlab_version/face_detection/mtcnn/setup.m @@ -0,0 +1,23 @@ +function setup(varargin) + +addpath C:\matconvnet\matconvnet-1.0-beta25\examples; + +opts.useGpu = false ; +opts.verbose = false ; +opts = vl_argparse(opts, varargin) ; + +try + vl_nnconv(single(1),single(1),[]) ; +catch + warning('VL_NNCONV() does not seem to be compiled. Trying to compile it now.') ; + vl_compilenn('enableGpu', opts.useGpu, 'verbose', opts.verbose) ; +end + +if opts.useGpu + try + vl_nnconv(gpuArray(single(1)),gpuArray(single(1)),[]) ; + catch + vl_compilenn('enableGpu', opts.useGpu, 'verbose', opts.verbose) ; + warning('GPU support does not seem to be compiled in MatConvNet. Trying to compile it now') ; + end +end