More visualization cleanup and removal of redundant code.
This commit is contained in:
Tadas Baltrusaitis
parent
e1c260eee6
commit
5ce0a8d4bf
11 changed files with 80 additions and 261 deletions
|
|
@ -206,37 +206,17 @@ int main (int argc, char **argv)
|
|||
// Displaying the tracking visualizations
|
||||
visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
|
||||
visualizer.SetObservationFaceAlign(sim_warped_img);
|
||||
visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model));
|
||||
visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy));
|
||||
visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols);
|
||||
visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, detection_success);
|
||||
visualizer.SetObservationPose(pose_estimate);
|
||||
visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
|
||||
visualizer.ShowObservation();
|
||||
|
||||
if(!det_parameters.quiet_mode)
|
||||
{
|
||||
if (recording_params.outputTrackedVideo() || visualize_track)
|
||||
{
|
||||
cv::namedWindow("tracking_result", 1);
|
||||
cv::imshow("tracking_result", captured_image);
|
||||
cv::waitKey(1);
|
||||
}
|
||||
if (visualize_align)
|
||||
{
|
||||
cv::imshow("sim_warp", sim_warped_img);
|
||||
}
|
||||
if (visualize_hog)
|
||||
{
|
||||
cv::Mat_<double> hog_descriptor_vis;
|
||||
FaceAnalysis::Visualise_FHOG(hog_descriptor, num_hog_rows, num_hog_cols, hog_descriptor_vis);
|
||||
cv::imshow("hog", hog_descriptor_vis);
|
||||
}
|
||||
}
|
||||
|
||||
// Setting up the recorder output
|
||||
open_face_rec.SetObservationHOG(detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG
|
||||
open_face_rec.SetObservationVisualization(visualizer.GetVisImage());
|
||||
open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
|
||||
open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model));
|
||||
open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy));
|
||||
open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy),
|
||||
face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success);
|
||||
open_face_rec.SetObservationPose(pose_estimate);
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@
|
|||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<SDLCheck>true</SDLCheck>
|
||||
<AdditionalIncludeDirectories>./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
|
||||
</ClCompile>
|
||||
</ItemDefinitionGroup>
|
||||
|
|
@ -100,7 +100,7 @@
|
|||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<SDLCheck>true</SDLCheck>
|
||||
<AdditionalIncludeDirectories>./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<PreprocessorDefinitions>WIN64;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
|
||||
</ClCompile>
|
||||
|
|
@ -113,7 +113,7 @@
|
|||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<SDLCheck>
|
||||
</SDLCheck>
|
||||
<AdditionalIncludeDirectories>./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<MultiProcessorCompilation>true</MultiProcessorCompilation>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
|
||||
|
|
@ -131,7 +131,7 @@
|
|||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<SDLCheck>
|
||||
</SDLCheck>
|
||||
<AdditionalIncludeDirectories>./include;../LandmarkDetector/include;../Utilities/include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>./include;../LandmarkDetector/include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<MultiProcessorCompilation>true</MultiProcessorCompilation>
|
||||
<PreprocessorDefinitions>WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnableEnhancedInstructionSet>AdvancedVectorExtensions</EnableEnhancedInstructionSet>
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@
|
|||
|
||||
#include "LandmarkDetectorUtils.h"
|
||||
#include "LandmarkDetectorFunc.h"
|
||||
#include "Utilities.h"
|
||||
#include "RotationHelpers.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
|
|
@ -185,12 +185,12 @@ void GazeAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_mode
|
|||
|
||||
cv::Mat_<double> proj_points;
|
||||
cv::Mat_<double> mesh_0 = (cv::Mat_<double>(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z);
|
||||
LandmarkDetector::Project(proj_points, mesh_0, fx, fy, cx, cy);
|
||||
Utilities::Project(proj_points, mesh_0, fx, fy, cx, cy);
|
||||
cv::line(img, cv::Point(cvRound(proj_points.at<double>(0,0) * (double)gaze_draw_multiplier), cvRound(proj_points.at<double>(0, 1) * (double)gaze_draw_multiplier)),
|
||||
cv::Point(cvRound(proj_points.at<double>(1, 0) * (double)gaze_draw_multiplier), cvRound(proj_points.at<double>(1, 1) * (double)gaze_draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, gaze_draw_shiftbits);
|
||||
|
||||
cv::Mat_<double> mesh_1 = (cv::Mat_<double>(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z);
|
||||
LandmarkDetector::Project(proj_points, mesh_1, fx, fy, cx, cy);
|
||||
Utilities::Project(proj_points, mesh_1, fx, fy, cx, cy);
|
||||
cv::line(img, cv::Point(cvRound(proj_points.at<double>(0, 0) * (double)gaze_draw_multiplier), cvRound(proj_points.at<double>(0, 1) * (double)gaze_draw_multiplier)),
|
||||
cv::Point(cvRound(proj_points.at<double>(1, 0) * (double)gaze_draw_multiplier), cvRound(proj_points.at<double>(1, 1) * (double)gaze_draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, gaze_draw_shiftbits);
|
||||
}
|
||||
|
|
@ -54,8 +54,6 @@ namespace LandmarkDetector
|
|||
//=============================================================================================
|
||||
void get_video_input_output_params(vector<string> &input_video_file, vector<string> &output_files, string &output_codec, vector<string> &arguments);
|
||||
|
||||
void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector<string> &arguments);
|
||||
|
||||
void get_image_input_output_params(vector<string> &input_image_files, vector<string> &output_feature_files, vector<string> &output_pose_files, vector<string> &output_image_files,
|
||||
vector<cv::Rect_<double>> &input_bounding_boxes, vector<string> &arguments);
|
||||
|
||||
|
|
@ -88,11 +86,7 @@ namespace LandmarkDetector
|
|||
vector<cv::Point2d> CalculateAllLandmarks(const cv::Mat_<double>& shape2D);
|
||||
vector<cv::Point2d> CalculateAllLandmarks(const CLNF& clnf_model);
|
||||
vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model);
|
||||
void DrawLandmarks(cv::Mat img, vector<cv::Point> landmarks);
|
||||
|
||||
void Draw(cv::Mat img, const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities);
|
||||
void Draw(cv::Mat img, const cv::Mat_<double>& shape2D);
|
||||
void Draw(cv::Mat img, const CLNF& clnf_model);
|
||||
vector<cv::Point3d> Calculate3DEyeLandmarks(const CLNF& clnf_model, double fx, double fy, double cx, double cy);
|
||||
|
||||
//============================================================================
|
||||
// Face detection helpers
|
||||
|
|
|
|||
|
|
@ -847,152 +847,6 @@ vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model)
|
|||
return to_return;
|
||||
}
|
||||
|
||||
|
||||
// Drawing landmarks on a face image
|
||||
void Draw(cv::Mat img, const cv::Mat_<double>& shape2D, const cv::Mat_<int>& visibilities)
|
||||
{
|
||||
int n = shape2D.rows/2;
|
||||
|
||||
|
||||
// Drawing feature points
|
||||
if(n >= 66)
|
||||
{
|
||||
for( int i = 0; i < n; ++i)
|
||||
{
|
||||
if(visibilities.at<int>(i))
|
||||
{
|
||||
cv::Point featurePoint(cvRound(shape2D.at<double>(i) * (double)draw_multiplier), cvRound(shape2D.at<double>(i + n) * (double)draw_multiplier));
|
||||
|
||||
// A rough heuristic for drawn point size
|
||||
int thickness = (int)std::ceil(3.0* ((double)img.cols) / 640.0);
|
||||
int thickness_2 = (int)std::ceil(1.0* ((double)img.cols) / 640.0);
|
||||
|
||||
cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits);
|
||||
cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
else if(n == 28) // drawing eyes
|
||||
{
|
||||
for( int i = 0; i < n; ++i)
|
||||
{
|
||||
cv::Point featurePoint(cvRound(shape2D.at<double>(i) * (double)draw_multiplier), cvRound(shape2D.at<double>(i + n) * (double)draw_multiplier));
|
||||
|
||||
// A rough heuristic for drawn point size
|
||||
int thickness = 1.0;
|
||||
int thickness_2 = 1.0;
|
||||
|
||||
int next_point = i + 1;
|
||||
if(i == 7)
|
||||
next_point = 0;
|
||||
if(i == 19)
|
||||
next_point = 8;
|
||||
if(i == 27)
|
||||
next_point = 20;
|
||||
|
||||
cv::Point nextFeaturePoint(cvRound(shape2D.at<double>(next_point) * (double)draw_multiplier), cvRound(shape2D.at<double>(next_point + n) * (double)draw_multiplier));
|
||||
if( i < 8 || i > 19)
|
||||
cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits);
|
||||
else
|
||||
cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA, draw_shiftbits);
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
else if(n == 6)
|
||||
{
|
||||
for( int i = 0; i < n; ++i)
|
||||
{
|
||||
cv::Point featurePoint(cvRound(shape2D.at<double>(i) * (double)draw_multiplier), cvRound(shape2D.at<double>(i + n) * (double)draw_multiplier));
|
||||
|
||||
// A rough heuristic for drawn point size
|
||||
int thickness = 1.0;
|
||||
int thickness_2 = 1.0;
|
||||
|
||||
int next_point = i + 1;
|
||||
if(i == 5)
|
||||
next_point = 0;
|
||||
|
||||
cv::Point nextFeaturePoint(cvRound(shape2D.at<double>(next_point) * (double)draw_multiplier), cvRound(shape2D.at<double>(next_point + n) * (double)draw_multiplier));
|
||||
cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Drawing landmarks on a face image
|
||||
void Draw(cv::Mat img, const cv::Mat_<double>& shape2D)
|
||||
{
|
||||
|
||||
int n;
|
||||
|
||||
if(shape2D.cols == 2)
|
||||
{
|
||||
n = shape2D.rows;
|
||||
}
|
||||
else if(shape2D.cols == 1)
|
||||
{
|
||||
n = shape2D.rows/2;
|
||||
}
|
||||
|
||||
for( int i = 0; i < n; ++i)
|
||||
{
|
||||
cv::Point featurePoint;
|
||||
if(shape2D.cols == 1)
|
||||
{
|
||||
featurePoint = cv::Point(cvRound(shape2D.at<double>(i) * (double)draw_multiplier), cvRound(shape2D.at<double>(i + n) * (double)draw_multiplier));
|
||||
}
|
||||
else
|
||||
{
|
||||
featurePoint = cv::Point(cvRound(shape2D.at<double>(i, 0) * (double)draw_multiplier), cvRound(shape2D.at<double>(i, 1) * (double)draw_multiplier));
|
||||
}
|
||||
// A rough heuristic for drawn point size
|
||||
int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0);
|
||||
int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0);
|
||||
|
||||
cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits);
|
||||
cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Drawing detected landmarks on a face image
|
||||
void Draw(cv::Mat img, const CLNF& clnf_model)
|
||||
{
|
||||
|
||||
int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0);
|
||||
|
||||
// Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation
|
||||
Draw(img, clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]);
|
||||
|
||||
// If the model has hierarchical updates draw those too
|
||||
for(size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
|
||||
{
|
||||
if(clnf_model.hierarchical_models[i].pdm.NumberOfPoints() != clnf_model.hierarchical_mapping[i].size())
|
||||
{
|
||||
Draw(img, clnf_model.hierarchical_models[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void DrawLandmarks(cv::Mat img, vector<cv::Point> landmarks)
|
||||
{
|
||||
for(cv::Point p : landmarks)
|
||||
{
|
||||
|
||||
// A rough heuristic for drawn point size
|
||||
int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0);
|
||||
int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0);
|
||||
|
||||
cv::circle(img, p, 1, cv::Scalar(0,0,255), thickness, CV_AA);
|
||||
cv::circle(img, p, 1, cv::Scalar(255,0,0), thickness_2, CV_AA);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
//===========================================================================
|
||||
|
||||
//============================================================================
|
||||
|
|
|
|||
|
|
@ -114,5 +114,47 @@ namespace Utilities
|
|||
cv::Rodrigues(rotation_matrix, axis_angle);
|
||||
return axis_angle;
|
||||
}
|
||||
|
||||
// Generally useful 3D functions
|
||||
static void Project(cv::Mat_<double>& dest, const cv::Mat_<double>& mesh, double fx, double fy, double cx, double cy)
|
||||
{
|
||||
dest = cv::Mat_<double>(mesh.rows, 2, 0.0);
|
||||
|
||||
int num_points = mesh.rows;
|
||||
|
||||
double X, Y, Z;
|
||||
|
||||
cv::Mat_<double>::const_iterator mData = mesh.begin();
|
||||
cv::Mat_<double>::iterator projected = dest.begin();
|
||||
|
||||
for (int i = 0; i < num_points; i++)
|
||||
{
|
||||
// Get the points
|
||||
X = *(mData++);
|
||||
Y = *(mData++);
|
||||
Z = *(mData++);
|
||||
|
||||
double x;
|
||||
double y;
|
||||
|
||||
// if depth is 0 the projection is different
|
||||
if (Z != 0)
|
||||
{
|
||||
x = ((X * fx / Z) + cx);
|
||||
y = ((Y * fy / Z) + cy);
|
||||
}
|
||||
else
|
||||
{
|
||||
x = X;
|
||||
y = Y;
|
||||
}
|
||||
|
||||
// Project and store in dest matrix
|
||||
(*projected++) = x;
|
||||
(*projected++) = y;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
#endif
|
||||
|
|
@ -43,8 +43,6 @@ namespace Utilities
|
|||
|
||||
// TODO draw AU results
|
||||
|
||||
// Helper utilities
|
||||
void Project(cv::Mat_<double>& dest, const cv::Mat_<double>& mesh, double fx, double fy, double cx, double cy);
|
||||
|
||||
// Drawing a bounding box around the face in an image
|
||||
void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy);
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ namespace Utilities
|
|||
Visualizer(std::vector<std::string> arguments);
|
||||
Visualizer(bool vis_track, bool vis_hog, bool vis_align);
|
||||
|
||||
~Visualizer();
|
||||
//~Visualizer();
|
||||
|
||||
// TODO copy, assignment and move operators? Do not allow
|
||||
|
||||
|
|
|
|||
|
|
@ -157,7 +157,7 @@ void RecorderOpenFace::WriteObservation()
|
|||
{
|
||||
// As we are writing out the header, work out some things like number of landmarks, names of AUs etc.
|
||||
int num_face_landmarks = landmarks_2D.rows / 2;
|
||||
int num_eye_landmarks = (int)eye_landmarks.size();
|
||||
int num_eye_landmarks = (int)eye_landmarks2D.size();
|
||||
int num_model_modes = pdm_params_local.rows;
|
||||
|
||||
std::vector<std::string> au_names_class;
|
||||
|
|
@ -182,7 +182,7 @@ void RecorderOpenFace::WriteObservation()
|
|||
|
||||
this->csv_recorder.WriteLine(observation_count, timestamp, landmark_detection_success,
|
||||
landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose,
|
||||
gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks, au_intensities, au_occurences);
|
||||
gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks2D, eye_landmarks3D, au_intensities, au_occurences);
|
||||
|
||||
if(params.outputHOG())
|
||||
{
|
||||
|
|
|
|||
|
|
@ -43,46 +43,6 @@
|
|||
|
||||
namespace Utilities
|
||||
{
|
||||
void Project(cv::Mat_<double>& dest, const cv::Mat_<double>& mesh, double fx, double fy, double cx, double cy)
|
||||
{
|
||||
dest = cv::Mat_<double>(mesh.rows, 2, 0.0);
|
||||
|
||||
int num_points = mesh.rows;
|
||||
|
||||
double X, Y, Z;
|
||||
|
||||
|
||||
cv::Mat_<double>::const_iterator mData = mesh.begin();
|
||||
cv::Mat_<double>::iterator projected = dest.begin();
|
||||
|
||||
for (int i = 0; i < num_points; i++)
|
||||
{
|
||||
// Get the points
|
||||
X = *(mData++);
|
||||
Y = *(mData++);
|
||||
Z = *(mData++);
|
||||
|
||||
double x;
|
||||
double y;
|
||||
|
||||
// if depth is 0 the projection is different
|
||||
if (Z != 0)
|
||||
{
|
||||
x = ((X * fx / Z) + cx);
|
||||
y = ((Y * fy / Z) + cy);
|
||||
}
|
||||
else
|
||||
{
|
||||
x = X;
|
||||
y = Y;
|
||||
}
|
||||
|
||||
// Project and store in dest matrix
|
||||
(*projected++) = x;
|
||||
(*projected++) = y;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@
|
|||
|
||||
#include "Visualizer.h"
|
||||
#include "VisualizationUtils.h"
|
||||
#include "RotationHelpers.h"
|
||||
|
||||
// For drawing on images
|
||||
#include <opencv2/imgproc.hpp>
|
||||
|
|
@ -149,23 +150,22 @@ void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence)
|
|||
}
|
||||
}
|
||||
|
||||
// TODO add 3D eye landmark locations
|
||||
// Eye gaze infomration drawing, first of eye landmarks then of gaze
|
||||
void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d)
|
||||
{
|
||||
// TODO actual drawing, first of eye landmarks then of gaze
|
||||
|
||||
if (eye_landmarks.size() > 0)
|
||||
if (eye_landmarks2d.size() > 0)
|
||||
{
|
||||
// FIrst draw the eye region landmarks
|
||||
for (size_t i = 0; i < eye_landmarks.size(); ++i)
|
||||
// First draw the eye region landmarks
|
||||
for (size_t i = 0; i < eye_landmarks2d.size(); ++i)
|
||||
{
|
||||
cv::Point featurePoint(cvRound(eye_landmarks[i].x * (double)draw_multiplier), eye_landmarks[i].y * (double)draw_multiplier));
|
||||
cv::Point featurePoint(cvRound(eye_landmarks2d[i].x * (double)draw_multiplier), cvRound(eye_landmarks2d[i].y * (double)draw_multiplier));
|
||||
|
||||
// A rough heuristic for drawn point size
|
||||
int thickness = 1.0;
|
||||
int thickness_2 = 1.0;
|
||||
int thickness = 1;
|
||||
int thickness_2 = 1;
|
||||
|
||||
int next_point = i + 1;
|
||||
size_t next_point = i + 1;
|
||||
if (i == 7)
|
||||
next_point = 0;
|
||||
if (i == 19)
|
||||
|
|
@ -173,7 +173,7 @@ void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv
|
|||
if (i == 27)
|
||||
next_point = 20;
|
||||
|
||||
cv::Point nextFeaturePoint(cvRound(eye_landmarks[next_point].x * (double)draw_multiplier), cvRound(eye_landmarks[next_point].y * (double)draw_multiplier));
|
||||
cv::Point nextFeaturePoint(cvRound(eye_landmarks2d[next_point].x * (double)draw_multiplier), cvRound(eye_landmarks2d[next_point].y * (double)draw_multiplier));
|
||||
if (i < 8 || i > 19)
|
||||
cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits);
|
||||
else
|
||||
|
|
@ -184,36 +184,24 @@ void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv
|
|||
// Now draw the gaze lines themselves
|
||||
cv::Mat cameraMat = (cv::Mat_<double>(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0);
|
||||
|
||||
int part_left = -1;
|
||||
int part_right = -1;
|
||||
for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
|
||||
// Grabbing the pupil location, to draw eye gaze need to know where the pupil is
|
||||
cv::Point3d pupil_left(0, 0, 0);
|
||||
cv::Point3d pupil_right(0, 0, 0);
|
||||
for (size_t i = 0; i < 8; ++i)
|
||||
{
|
||||
if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0)
|
||||
{
|
||||
part_left = i;
|
||||
}
|
||||
if (clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0)
|
||||
{
|
||||
part_right = i;
|
||||
}
|
||||
pupil_left = pupil_left + eye_landmarks3d[i];
|
||||
pupil_right = pupil_right + eye_landmarks3d[i + eye_landmarks3d.size()];
|
||||
}
|
||||
|
||||
cv::Mat eyeLdmks3d_left = clnf_model.hierarchical_models[part_left].GetShape(fx, fy, cx, cy);
|
||||
cv::Point3f pupil_left = GetPupilPosition(eyeLdmks3d_left);
|
||||
|
||||
cv::Mat_<double> irisLdmks3d_left = eyeLdmks3d_left.rowRange(0, 8);
|
||||
cv::Point3f pupil_left(cv::mean(irisLdmks3d_left.col(0))[0], cv::mean(irisLdmks3d_left.col(1))[0], cv::mean(irisLdmks3d_left.col(2))[0]);
|
||||
|
||||
cv::Mat eyeLdmks3d_right = clnf_model.hierarchical_models[part_right].GetShape(fx, fy, cx, cy);
|
||||
cv::Point3f pupil_right = GetPupilPosition(eyeLdmks3d_right);
|
||||
pupil_left = pupil_left / 8;
|
||||
pupil_right = pupil_right / 8;
|
||||
|
||||
std::vector<cv::Point3d> points_left;
|
||||
points_left.push_back(cv::Point3d(pupil_left));
|
||||
points_left.push_back(cv::Point3d(pupil_left + gaze_direction0*50.0));
|
||||
points_left.push_back(cv::Point3d(pupil_left + cv::Point3d(gaze_direction0)*50.0));
|
||||
|
||||
std::vector<cv::Point3d> points_right;
|
||||
points_right.push_back(cv::Point3d(pupil_right));
|
||||
points_right.push_back(cv::Point3d(pupil_right + gaze_direction1*50.0));
|
||||
points_right.push_back(cv::Point3d(pupil_right + cv::Point3d(gaze_direction1)*50.0));
|
||||
|
||||
cv::Mat_<double> proj_points;
|
||||
cv::Mat_<double> mesh_0 = (cv::Mat_<double>(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z);
|
||||
|
|
@ -248,4 +236,7 @@ void Visualizer::ShowObservation()
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
cv::Mat Visualizer::GetVisImage()
|
||||
{
|
||||
return captured_image;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue