r/sustaining_gazes
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Adding visualization utilities files.

Browse source
This commit is contained in:
Tadas Baltrusaitis 2017-11-11 10:08:02 +00:00
parent 4f890a7d42
commit 42bcb55d28
2 changed files with 228 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
lib/local/Utilities/include/VisualizationUtils.h Normal file
View file

@ -0,0 +1,63 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace: an open source facial behavior analysis toolkit
// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency
// in IEEE Winter Conference on Applications of Computer Vision, 2016
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
// Constrained Local Neural Fields for robust facial landmark detection in the wild.
// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency.
// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013.
//
///////////////////////////////////////////////////////////////////////////////
#ifndef __VISUALIZATION_UTILS_h_
#define __VISUALIZATION_UTILS_h_
#include <opencv2/core/core.hpp>
#include <vector>
namespace Utilities
{
void DrawLandmarkDetResults(cv::Mat img, const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities);
void DrawPoseDetResults(cv::Mat image, cv::Vec6d pose, double confidence, float fx, float fy, float cx, float cy);
void DrawEyeTrackResults(cv::Mat image, const cv::Mat_<double>& eye_landmarks, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy);
// TODO draw AU results
// Helper utilities
void Project(cv::Mat_<double>& dest, const cv::Mat_<double>& mesh, double fx, double fy, double cx, double cy);
// Drawing a bounding box around the face in an image
void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy);
void DrawBox(std::vector<std::pair<cv::Point, cv::Point>> lines, cv::Mat image, cv::Scalar color, int thickness);
// Computing a bounding box to be drawn
std::vector<std::pair<cv::Point2d, cv::Point2d>> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy);
}
#endif

165
lib/local/Utilities/src/VisualizationUtils.cpp Normal file
View file

@ -0,0 +1,165 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace: an open source facial behavior analysis toolkit
// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency
// in IEEE Winter Conference on Applications of Computer Vision, 2016
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
// Constrained Local Neural Fields for robust facial landmark detection in the wild.
// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency.
// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013.
//
///////////////////////////////////////////////////////////////////////////////
#include "VisualizationUtils.h"
using namespace Utilities;
void Project(cv::Mat_<double>& dest, const cv::Mat_<double>& mesh, double fx, double fy, double cx, double cy)
{
dest = cv::Mat_<double>(mesh.rows, 2, 0.0);
int num_points = mesh.rows;
double X, Y, Z;
cv::Mat_<double>::const_iterator mData = mesh.begin();
cv::Mat_<double>::iterator projected = dest.begin();
for (int i = 0; i < num_points; i++)
{
// Get the points
X = *(mData++);
Y = *(mData++);
Z = *(mData++);
double x;
double y;
// if depth is 0 the projection is different
if (Z != 0)
{
x = ((X * fx / Z) + cx);
y = ((Y * fy / Z) + cy);
}
else
{
x = X;
y = Y;
}
// Project and store in dest matrix
(*projected++) = x;
(*projected++) = y;
}
}
void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy)
{
auto edge_lines = CalculateBox(pose, fx, fy, cx, cy);
DrawBox(edge_lines, image, color, thickness);
}
std::vector<std::pair<cv::Point2d, cv::Point2d>> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy)
{
double boxVerts[] = { -1, 1, -1,
1, 1, -1,
1, 1, 1,
-1, 1, 1,
1, -1, 1,
1, -1, -1,
-1, -1, -1,
-1, -1, 1 };
std::vector<std::pair<int, int>> edges;
edges.push_back(std::pair<int, int>(0, 1));
edges.push_back(std::pair<int, int>(1, 2));
edges.push_back(std::pair<int, int>(2, 3));
edges.push_back(std::pair<int, int>(0, 3));
edges.push_back(std::pair<int, int>(2, 4));
edges.push_back(std::pair<int, int>(1, 5));
edges.push_back(std::pair<int, int>(0, 6));
edges.push_back(std::pair<int, int>(3, 7));
edges.push_back(std::pair<int, int>(6, 5));
edges.push_back(std::pair<int, int>(5, 4));
edges.push_back(std::pair<int, int>(4, 7));
edges.push_back(std::pair<int, int>(7, 6));
// The size of the head is roughly 200mm x 200mm x 200mm
cv::Mat_<double> box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100;
cv::Matx33d rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5]));
cv::Mat_<double> rotBox;
// Rotate the box
rotBox = cv::Mat(rot) * box.t();
rotBox = rotBox.t();
// Move the bounding box to head position
rotBox.col(0) = rotBox.col(0) + pose[0];
rotBox.col(1) = rotBox.col(1) + pose[1];
rotBox.col(2) = rotBox.col(2) + pose[2];
// draw the lines
cv::Mat_<double> rotBoxProj;
Project(rotBoxProj, rotBox, fx, fy, cx, cy);
std::vector<std::pair<cv::Point2d, cv::Point2d>> lines;
for (size_t i = 0; i < edges.size(); ++i)
{
cv::Mat_<double> begin;
cv::Mat_<double> end;
rotBoxProj.row(edges[i].first).copyTo(begin);
rotBoxProj.row(edges[i].second).copyTo(end);
cv::Point2d p1(begin.at<double>(0), begin.at<double>(1));
cv::Point2d p2(end.at<double>(0), end.at<double>(1));
lines.push_back(std::pair<cv::Point2d, cv::Point2d>(p1, p2));
}
return lines;
}
void DrawBox(std::vector<std::pair<cv::Point, cv::Point>> lines, cv::Mat image, cv::Scalar color, int thickness)
{
cv::Rect image_rect(0, 0, image.cols, image.rows);
for (size_t i = 0; i < lines.size(); ++i)
{
cv::Point p1 = lines.at(i).first;
cv::Point p2 = lines.at(i).second;
// Only draw the line if one of the points is inside the image
if (p1.inside(image_rect) || p2.inside(image_rect))
{
cv::line(image, p1, p2, color, thickness, CV_AA);
}
}
}