From 42bcb55d2869570e653060b2db3a8c3a8eda2200 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Sat, 11 Nov 2017 10:08:02 +0000 Subject: [PATCH] Adding visualization utilities files. --- .../Utilities/include/VisualizationUtils.h | 63 +++++++ .../Utilities/src/VisualizationUtils.cpp | 165 ++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 lib/local/Utilities/include/VisualizationUtils.h create mode 100644 lib/local/Utilities/src/VisualizationUtils.cpp diff --git a/lib/local/Utilities/include/VisualizationUtils.h b/lib/local/Utilities/include/VisualizationUtils.h new file mode 100644 index 0000000..fcd3722 --- /dev/null +++ b/lib/local/Utilities/include/VisualizationUtils.h @@ -0,0 +1,63 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __VISUALIZATION_UTILS_h_ +#define __VISUALIZATION_UTILS_h_ + +#include + +#include + +namespace Utilities +{ + + void DrawLandmarkDetResults(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities); + void DrawPoseDetResults(cv::Mat image, cv::Vec6d pose, double confidence, float fx, float fy, float cx, float cy); + void DrawEyeTrackResults(cv::Mat image, const cv::Mat_& eye_landmarks, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy); + + // TODO draw AU results + + // Helper utilities + void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy); + + // Drawing a bounding box around the face in an image + void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy); + void DrawBox(std::vector> lines, cv::Mat image, cv::Scalar color, int thickness); + + // Computing a bounding box to be drawn + std::vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); + + + +} +#endif \ No newline at end of file diff --git a/lib/local/Utilities/src/VisualizationUtils.cpp b/lib/local/Utilities/src/VisualizationUtils.cpp new file mode 100644 index 0000000..8df8bc4 --- /dev/null +++ b/lib/local/Utilities/src/VisualizationUtils.cpp @@ -0,0 +1,165 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "VisualizationUtils.h" + +using namespace Utilities; + +void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy) +{ + dest = cv::Mat_(mesh.rows, 2, 0.0); + + int num_points = mesh.rows; + + double X, Y, Z; + + + cv::Mat_::const_iterator mData = mesh.begin(); + cv::Mat_::iterator projected = dest.begin(); + + for (int i = 0; i < num_points; i++) + { + // Get the points + X = *(mData++); + Y = *(mData++); + Z = *(mData++); + + double x; + double y; + + // if depth is 0 the projection is different + if (Z != 0) + { + x = ((X * fx / Z) + cx); + y = ((Y * fy / Z) + cy); + } + else + { + x = X; + y = Y; + } + + // Project and store in dest matrix + (*projected++) = x; + (*projected++) = y; + } + +} + +void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy) +{ + auto edge_lines = CalculateBox(pose, fx, fy, cx, cy); + DrawBox(edge_lines, image, color, thickness); +} + +std::vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy) +{ + double boxVerts[] = { -1, 1, -1, + 1, 1, -1, + 1, 1, 1, + -1, 1, 1, + 1, -1, 1, + 1, -1, -1, + -1, -1, -1, + -1, -1, 1 }; + + std::vector> edges; + edges.push_back(std::pair(0, 1)); + edges.push_back(std::pair(1, 2)); + edges.push_back(std::pair(2, 3)); + edges.push_back(std::pair(0, 3)); + edges.push_back(std::pair(2, 4)); + edges.push_back(std::pair(1, 5)); + edges.push_back(std::pair(0, 6)); + edges.push_back(std::pair(3, 7)); + edges.push_back(std::pair(6, 5)); + edges.push_back(std::pair(5, 4)); + edges.push_back(std::pair(4, 7)); + edges.push_back(std::pair(7, 6)); + + // The size of the head is roughly 200mm x 200mm x 200mm + cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100; + + cv::Matx33d rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5])); + cv::Mat_ rotBox; + + // Rotate the box + rotBox = cv::Mat(rot) * box.t(); + rotBox = rotBox.t(); + + // Move the bounding box to head position + rotBox.col(0) = rotBox.col(0) + pose[0]; + rotBox.col(1) = rotBox.col(1) + pose[1]; + rotBox.col(2) = rotBox.col(2) + pose[2]; + + // draw the lines + cv::Mat_ rotBoxProj; + Project(rotBoxProj, rotBox, fx, fy, cx, cy); + + std::vector> lines; + + for (size_t i = 0; i < edges.size(); ++i) + { + cv::Mat_ begin; + cv::Mat_ end; + + rotBoxProj.row(edges[i].first).copyTo(begin); + rotBoxProj.row(edges[i].second).copyTo(end); + + cv::Point2d p1(begin.at(0), begin.at(1)); + cv::Point2d p2(end.at(0), end.at(1)); + + lines.push_back(std::pair(p1, p2)); + + } + + return lines; +} + +void DrawBox(std::vector> lines, cv::Mat image, cv::Scalar color, int thickness) +{ + cv::Rect image_rect(0, 0, image.cols, image.rows); + + for (size_t i = 0; i < lines.size(); ++i) + { + cv::Point p1 = lines.at(i).first; + cv::Point p2 = lines.at(i).second; + // Only draw the line if one of the points is inside the image + if (p1.inside(image_rect) || p2.inside(image_rect)) + { + cv::line(image, p1, p2, color, thickness, CV_AA); + } + + } + +}