From c1431fe815fb65dc67be86664158a475e32d9c20 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Tue, 6 Sep 2016 10:20:31 -0400 Subject: [PATCH] Fixes with drawing antialiased lines and landmarks using sub-pixel precision. --- lib/local/FaceAnalyser/src/GazeEstimation.cpp | 10 ++- .../include/LandmarkDetectorUtils.h | 2 +- .../src/LandmarkDetectorUtils.cpp | 61 ++++++++++--------- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/lib/local/FaceAnalyser/src/GazeEstimation.cpp b/lib/local/FaceAnalyser/src/GazeEstimation.cpp index 9ac0896..0278f1a 100644 --- a/lib/local/FaceAnalyser/src/GazeEstimation.cpp +++ b/lib/local/FaceAnalyser/src/GazeEstimation.cpp @@ -68,6 +68,10 @@ using namespace std; using namespace FaceAnalysis; +// For subpixel accuracy drawing +const int gaze_draw_shiftbits = 4; +const int gaze_draw_multiplier = 1 << 4; + cv::Point3f RaySphereIntersect(cv::Point3f rayOrigin, cv::Point3f rayDir, cv::Point3f sphereOrigin, float sphereRadius){ float dx = rayDir.x; @@ -190,9 +194,11 @@ void FaceAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_mode cv::Mat_ proj_points; cv::Mat_ mesh_0 = (cv::Mat_(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z); LandmarkDetector::Project(proj_points, mesh_0, fx, fy, cx, cy); - line(img, cv::Point(proj_points.at(0,0), proj_points.at(0, 1)), cv::Point(proj_points.at(1, 0), proj_points.at(1, 1)), cv::Scalar(110, 220, 0), 2, 8); + cv::line(img, cv::Point(cvRound(proj_points.at(0,0) * gaze_draw_multiplier), cvRound(proj_points.at(0, 1) * gaze_draw_multiplier)), + cv::Point(cvRound(proj_points.at(1, 0) * gaze_draw_multiplier), cvRound(proj_points.at(1, 1) * gaze_draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, gaze_draw_shiftbits); cv::Mat_ mesh_1 = (cv::Mat_(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z); LandmarkDetector::Project(proj_points, mesh_1, fx, fy, cx, cy); - line(img, cv::Point(proj_points.at(0, 0), proj_points.at(0, 1)), cv::Point(proj_points.at(1, 0), proj_points.at(1, 1)), cv::Scalar(110, 220, 0), 2, 8); + cv::line(img, cv::Point(cvRound(proj_points.at(0, 0) * gaze_draw_multiplier), cvRound(proj_points.at(0, 1) * gaze_draw_multiplier)), + cv::Point(cvRound(proj_points.at(1, 0) * gaze_draw_multiplier), cvRound(proj_points.at(1, 1) * gaze_draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, gaze_draw_shiftbits); } \ No newline at end of file diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index 87d2c5b..e74cf49 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -110,7 +110,7 @@ namespace LandmarkDetector void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy); // Drawing face bounding box - vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); + vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); void DrawBox(vector> lines, cv::Mat image, cv::Scalar color, int thickness); vector CalculateLandmarks(const cv::Mat_& shape2D, cv::Mat_& visibilities); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 5ebec43..fa19007 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -76,6 +76,11 @@ using namespace std; namespace LandmarkDetector { +// For subpixel accuracy drawing +const int draw_shiftbits = 4; +const int draw_multiplier = 1 << 4; + + // Useful utility for creating directories for storing the output files void create_directory_from_file(string output_path) { @@ -888,7 +893,7 @@ void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, flo cv::Mat_ rotBoxProj; Project(rotBoxProj, rotBox, fx, fy, cx, cy); - cv::Rect image_rect(0,0,image.cols, image.rows); + cv::Rect image_rect(0,0,image.cols * draw_multiplier, image.rows * draw_multiplier); for (size_t i = 0; i < edges.size(); ++i) { @@ -898,20 +903,21 @@ void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, flo rotBoxProj.row(edges[i].first).copyTo(begin); rotBoxProj.row(edges[i].second).copyTo(end); - cv::Point p1((int)begin.at(0), (int)begin.at(1)); - cv::Point p2((int)end.at(0), (int)end.at(1)); + + cv::Point p1(cvRound(begin.at(0) * draw_multiplier), cvRound(begin.at(1) * draw_multiplier)); + cv::Point p2(cvRound(end.at(0) * draw_multiplier), cvRound(end.at(1) * draw_multiplier)); // Only draw the line if one of the points is inside the image if(p1.inside(image_rect) || p2.inside(image_rect)) { - cv::line(image, p1, p2, color, thickness, CV_AA); + cv::line(image, p1, p2, color, thickness, CV_AA, draw_shiftbits); } } } -vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy) +vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy) { double boxVerts[] = {-1, 1, -1, 1, 1, -1, @@ -955,7 +961,7 @@ vector> CalculateBox(cv::Vec6d pose, float fx, f cv::Mat_ rotBoxProj; Project(rotBoxProj, rotBox, fx, fy, cx, cy); - vector> lines; + vector> lines; for (size_t i = 0; i < edges.size(); ++i) { @@ -965,10 +971,10 @@ vector> CalculateBox(cv::Vec6d pose, float fx, f rotBoxProj.row(edges[i].first).copyTo(begin); rotBoxProj.row(edges[i].second).copyTo(end); - cv::Point p1((int)begin.at(0), (int)begin.at(1)); - cv::Point p2((int)end.at(0), (int)end.at(1)); + cv::Point2d p1(begin.at(0), begin.at(1)); + cv::Point2d p2(end.at(0), end.at(1)); - lines.push_back(pair(p1,p2)); + lines.push_back(pair(p1,p2)); } @@ -1061,6 +1067,7 @@ vector CalculateLandmarks(CLNF& clnf_model) void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities) { int n = shape2D.rows/2; + // Drawing feature points if(n >= 66) @@ -1069,14 +1076,15 @@ void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& vis { if(visibilities.at(i)) { - cv::Point featurePoint((int)shape2D.at(i), (int)shape2D.at(i +n)); + cv::Point featurePoint(cvRound(shape2D.at(i) * draw_multiplier), cvRound(shape2D.at(i + n) * draw_multiplier)); // A rough heuristic for drawn point size int thickness = (int)std::ceil(3.0* ((double)img.cols) / 640.0); int thickness_2 = (int)std::ceil(1.0* ((double)img.cols) / 640.0); - cv::circle(img, featurePoint, 1, cv::Scalar(0,0,255), thickness, CV_AA); - cv::circle(img, featurePoint, 1, cv::Scalar(255,0,0), thickness_2, CV_AA); + cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); + cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); + } } } @@ -1084,7 +1092,7 @@ void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& vis { for( int i = 0; i < n; ++i) { - cv::Point featurePoint((int)shape2D.at(i), (int)shape2D.at(i +n)); + cv::Point featurePoint(cvRound(shape2D.at(i) * draw_multiplier), cvRound(shape2D.at(i + n) * draw_multiplier)); // A rough heuristic for drawn point size int thickness = 1.0; @@ -1098,15 +1106,12 @@ void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& vis if(i == 27) next_point = 20; - cv::Point nextFeaturePoint((int)shape2D.at(next_point), (int)shape2D.at(next_point+n)); + cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * draw_multiplier), cvRound(shape2D.at(next_point + n) * draw_multiplier)); if( i < 8 || i > 19) - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA); + cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); else - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA); + cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA, draw_shiftbits); - //cv::circle(img, featurePoint, 1, Scalar(0,255,0), thickness); - //cv::circle(img, featurePoint, 1, Scalar(0,0,255), thickness_2); - } } @@ -1114,21 +1119,18 @@ void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& vis { for( int i = 0; i < n; ++i) { - cv::Point featurePoint((int)shape2D.at(i), (int)shape2D.at(i +n)); + cv::Point featurePoint(cvRound(shape2D.at(i) * draw_multiplier), cvRound(shape2D.at(i + n) * draw_multiplier)); // A rough heuristic for drawn point size int thickness = 1.0; int thickness_2 = 1.0; - //cv::circle(img, featurePoint, 1, Scalar(0,255,0), thickness); - //cv::circle(img, featurePoint, 1, Scalar(0,0,255), thickness_2); - int next_point = i + 1; if(i == 5) next_point = 0; - cv::Point nextFeaturePoint((int)shape2D.at(next_point), (int)shape2D.at(next_point+n)); - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA); + cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * draw_multiplier), cvRound(shape2D.at(next_point + n) * draw_multiplier)); + cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); } } } @@ -1153,18 +1155,18 @@ void Draw(cv::Mat img, const cv::Mat_& shape2D) cv::Point featurePoint; if(shape2D.cols == 1) { - featurePoint = cv::Point((int)shape2D.at(i), (int)shape2D.at(i +n)); + featurePoint = cv::Point(cvRound(shape2D.at(i) * draw_multiplier), cvRound(shape2D.at(i + n) * draw_multiplier)); } else { - featurePoint = cv::Point((int)shape2D.at(i, 0), (int)shape2D.at(i, 1)); + featurePoint = cv::Point(cvRound(shape2D.at(i, 0) * draw_multiplier), cvRound(shape2D.at(i, 1) * draw_multiplier)); } // A rough heuristic for drawn point size int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0); - cv::circle(img, featurePoint, 1, cv::Scalar(0,0,255), thickness, CV_AA); - cv::circle(img, featurePoint, 1, cv::Scalar(255,0,0), thickness_2, CV_AA); + cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); + cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); } @@ -1193,6 +1195,7 @@ void DrawLandmarks(cv::Mat img, vector landmarks) { for(cv::Point p : landmarks) { + // A rough heuristic for drawn point size int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0);