affdex-sdk-cpp-samples/common/Visualizer.cpp

344 lines
12 KiB
C++
Raw Normal View History

#include "Visualizer.h"
#include <boost/format.hpp>
#include "affdex_small_logo.h"
#include <algorithm>
Visualizer::Visualizer():
GREEN_COLOR_CLASSIFIERS({
"joy"
}),
RED_COLOR_CLASSIFIERS({
"anger", "disgust", "sadness", "fear", "contempt"
})
{
logo_resized = false;
logo = cv::imdecode(cv::InputArray(small_logo), CV_LOAD_IMAGE_UNCHANGED);
EXPRESSIONS = {
"smile", "innerBrowRaise", "browRaise", "browFurrow", "noseWrinkle",
"upperLipRaise", "lipCornerDepressor", "chinRaise", "lipPucker", "lipPress",
"lipSuck", "mouthOpen", "smirk", "eyeClosure", "attention", "eyeWiden", "cheekRaise",
"lidTighten", "dimpler", "lipStretch", "jawDrop"
};
EMOTIONS = {
"joy", "fear", "disgust", "sadness", "anger",
"surprise", "contempt", "valence", "engagement"
};
HEAD_ANGLES = { "pitch", "yaw", "roll" };
EMOJIS = std::vector<std::string> {
"relaxed", "smiley", "laughing",
"kissing", "disappointed",
"rage", "smirk", "wink",
"stuckOutTongueWinkingEye", "stuckOutTongue",
"flushed", "scream"
};
GENDER_MAP = std::map<affdex::Gender, std::string> {
{ affdex::Gender::Male, "male" },
{ affdex::Gender::Female, "female" },
{ affdex::Gender::Unknown, "unknown" },
};
GLASSES_MAP = std::map<affdex::Glasses, std::string> {
{ affdex::Glasses::Yes, "yes" },
{ affdex::Glasses::No, "no" }
};
AGE_MAP = std::map<affdex::Age, std::string> {
{ affdex::Age::AGE_UNKNOWN, "unknown"},
{ affdex::Age::AGE_UNDER_18, "under 18" },
{ affdex::Age::AGE_18_24, "18-24" },
{ affdex::Age::AGE_25_34, "25-34" },
{ affdex::Age::AGE_35_44, "35-44" },
{ affdex::Age::AGE_45_54, "45-54" },
{ affdex::Age::AGE_55_64, "55-64" },
{ affdex::Age::AGE_65_PLUS, "65 plus" }
};
ETHNICITY_MAP = std::map<affdex::Ethnicity, std::string> {
{ affdex::Ethnicity::UNKNOWN, "unknown"},
{ affdex::Ethnicity::CAUCASIAN, "caucasian" },
{ affdex::Ethnicity::BLACK_AFRICAN, "black african" },
{ affdex::Ethnicity::SOUTH_ASIAN, "south asian" },
{ affdex::Ethnicity::EAST_ASIAN, "east asian" },
{ affdex::Ethnicity::HISPANIC, "hispanic" }
};
}
void Visualizer::drawFaceMetrics(affdex::Face face, std::vector<cv::Point2f> bounding_box)
{
cv::Scalar white_color = cv::Scalar(255, 255, 255);
//Draw Right side metrics
int padding = bounding_box[0].y; //Top left Y
drawValues((float *)&face.expressions, EXPRESSIONS,
bounding_box[2].x + spacing, padding, white_color, false);
padding = bounding_box[2].y; //Top left Y
//Draw Head Angles
drawHeadOrientation(face.measurements.orientation,
bounding_box[0].x - spacing, padding);
//Draw Appearance
drawAppearance(face.appearance, bounding_box[0].x - spacing, padding);
//Draw Left side metrics
drawValues((float *)&face.emotions, EMOTIONS,
bounding_box[0].x - spacing, padding, white_color, true);
}
void Visualizer::drawValues(const float * first, const std::vector<std::string> names,
const int x, int &padding, const cv::Scalar clr, const bool align_right)
{
for (std::string name : names)
{
drawClassifierOutput(name, (*first), cv::Point(x, padding += spacing), align_right);
first++;
}
}
void Visualizer::updateImage(cv::Mat output_img)
{
img = output_img;
if (!logo_resized)
{
double logo_width = (logo.size().width > img.size().width*0.25 ? img.size().width*0.25 : logo.size().width);
double logo_height = ((double)logo_width) * ((double)logo.size().height / logo.size().width);
cv::resize(logo, logo, cv::Size(logo_width, logo_height));
logo_resized = true;
}
cv::Mat roi = img(cv::Rect(img.cols - logo.cols - 10, 10, logo.cols, logo.rows));
overlayImage(logo, roi, cv::Point(0, 0));
}
void Visualizer::drawPoints(affdex::VecFeaturePoint points)
{
for (auto& point : points) //Draw face feature points.
{
cv::circle(img, cv::Point(point.x, point.y), 2.0f, cv::Scalar(255, 255, 255));
}
}
void Visualizer::drawBoundingBox(cv::Point2f top_left, cv::Point2f bottom_right, float valence)
{
//Draw bounding box
const ColorgenRedGreen valence_color_generator( -100, 100 );
cv::rectangle( img, top_left, bottom_right,
valence_color_generator(valence), 3);
}
/** @brief DrawText prints text on screen either right or left justified at the anchor location (loc)
* @param output_img -- Image we are plotting on
* @param name -- Name of the classifier
* @param value -- Value we are trying to display
* @param loc -- Exact location. When aligh_right is (true/false) this should be the (upper-right, upper-left)
* @param align_right -- Whether to right or left justify the text
* @param color -- Color
*/
void Visualizer::drawText(const std::string& name, const std::string& value,
const cv::Point2f loc, bool align_right, cv::Scalar color)
{
const int block_width = 8;
const int margin = 2;
const int block_size = 10;
const int max_blocks = 100/block_size;
cv::Point2f display_loc = loc;
const std::string label = name+": ";
if( align_right )
{
display_loc.x -= (margin+block_width) * max_blocks;
int baseline=0;
cv::Size txtSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5f, 5,&baseline);
display_loc.x -= txtSize.width;
}
cv::putText(img, label+value, display_loc, cv::FONT_HERSHEY_SIMPLEX, 0.5f, color, 1);
}
/** @brief DrawClassifierOutput handles choosing between equalizer or text as well as defining the colors
* @param name -- Name of the classifier
* @param value -- Value we are trying to display
* @param loc -- Exact location. When aligh_right is (true/false) this should be the (upper-right, upper-left)
* @param align_right -- Whether to right or left justify the text
*/
void Visualizer::drawClassifierOutput(const std::string& classifier,
const float value, const cv::Point2f& loc, bool align_right)
{
static const ColorgenLinear white_yellow_generator( 0, 100, cv::Scalar(255,255,255), cv::Scalar(0, 255, 255));
static const ColorgenRedGreen valence_color_generator( -100, 100 );
// Determine the display color
cv::Scalar color = cv::Scalar(255, 255, 255);
if( classifier == "valence")
{
color = valence_color_generator( value );
}
else if( RED_COLOR_CLASSIFIERS.count(classifier) )
{
color = cv::Scalar(0, 0, 255);
}
else if( GREEN_COLOR_CLASSIFIERS.count(classifier) )
{
color = cv::Scalar(0, 255, 0);
}
float equalizer_magnitude = value;
if( classifier == "valence" )
{
equalizer_magnitude = std::fabs(value);
}
drawEqualizer(classifier, equalizer_magnitude, loc, align_right, color );
}
void Visualizer::drawEqualizer(const std::string& name, const float value, const cv::Point2f& loc,
bool align_right, cv::Scalar color)
{
const int block_width = 8;
const int block_height = 10;
const int margin = 2;
const int block_size = 10;
const int max_blocks = 100/block_size;
int blocks = round(value / block_size);
int i = loc.x, j = loc.y - 10;
cv::Point2f display_loc = loc;
const std::string label = align_right? name+": " : " :"+name;
for (int x = 0 ; x < (100/block_size) ; x++)
{
cv::Scalar scalar_clr = color;
float alpha = 0.8;
const int ii = (std::max)( float(i), 0.0f);
const int jj = (std::max)( float(j), 0.0f);
const int width = (std::min)(float(block_width), float(img.size().width-ii));
const int height = (std::min)(float(block_height), float(img.size().height-jj));
if (height < 0 || width < 0) continue;
cv::Mat roi = img(cv::Rect(ii, jj, width, height));
if (x >= blocks)
{
alpha = 0.3;
scalar_clr = cv::Scalar(186, 186, 186);
}
cv::Mat color(roi.size(), CV_8UC3, scalar_clr);
cv::addWeighted(color, alpha, roi, 1.0 - alpha , 0.0, roi);
i += align_right? -(margin+block_width):(margin+block_width);
}
display_loc.x += align_right? -(margin+block_width) * max_blocks : (margin+block_width) * max_blocks;
if( align_right )
{
int baseline=0;
cv::Size txtSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5f, 5,&baseline);
display_loc.x -= txtSize.width;
}
cv::putText(img, label, display_loc, cv::FONT_HERSHEY_SIMPLEX, 0.5f, cv::Scalar(50,50,50), 5);
cv::putText(img, label, display_loc, cv::FONT_HERSHEY_SIMPLEX, 0.5f, cv::Scalar(255, 255, 255), 1);
}
void Visualizer::drawHeadOrientation(affdex::Orientation headAngles, const int x, int &padding,
bool align_right, cv::Scalar color)
{
std::string valueStr = boost::str(boost::format("%3.1f") % headAngles.pitch);
drawText("pitch", valueStr, cv::Point(x, padding += spacing), align_right, color );
valueStr = boost::str(boost::format("%3.1f") % headAngles.yaw);
drawText("yaw", valueStr, cv::Point(x, padding += spacing), align_right, color );
valueStr = boost::str(boost::format("%3.1f") % headAngles.roll);
drawText("roll", valueStr, cv::Point(x, padding += spacing), align_right, color );
}
void Visualizer::drawAppearance(affdex::Appearance appearance, const int x, int &padding,
bool align_right, cv::Scalar color)
{
drawText("gender", GENDER_MAP[appearance.gender], cv::Point(x, padding += spacing), align_right, color );
drawText("age", AGE_MAP[appearance.age], cv::Point(x, padding += spacing), align_right, color );
drawText("ethnicity", ETHNICITY_MAP[appearance.ethnicity], cv::Point(x, padding += spacing), align_right, color );
}
void Visualizer::showImage()
{
cv::imshow("analyze video", img);
cv::waitKey(5);
}
void Visualizer::overlayImage(const cv::Mat &foreground, cv::Mat &background, cv::Point2i location)
{
// start at the row indicated by location, or at row 0 if location.y is negative.
for(int y = (std::max)(location.y , 0); y < background.rows; ++y)
{
int fY = y - location.y; // because of the translation
// we are done of we have processed all rows of the foreground image.
if(fY >= foreground.rows)
break;
// start at the column indicated by location,
// or at column 0 if location.x is negative.
for(int x = (std::max)(location.x, 0); x < background.cols; ++x)
{
int fX = x - location.x; // because of the translation.
// we are done with this row if the column is outside of the foreground image.
if(fX >= foreground.cols)
break;
// determine the opacity of the foregrond pixel, using its fourth (alpha) channel.
double opacity =
((double)foreground.data[fY * foreground.step + fX * foreground.channels() + (foreground.channels()-1)])
/ 255.;
// and now combine the background and foreground pixel, using the opacity,
// but only if opacity > 0.
for(int c = 0; opacity > 0 && c < background.channels(); ++c)
{
unsigned char foregroundPx =
foreground.data[fY * foreground.step + fX * foreground.channels() + c];
unsigned char backgroundPx =
background.data[y * background.step + x * background.channels() + c];
background.data[y*background.step + background.channels()*x + c] =
backgroundPx * (1.-opacity) + foregroundPx * opacity;
}
}
}
}
cv::Scalar ColorgenRedGreen::operator()( const float val ) const
{
float norm_val = ( val - red_val_ ) / ( green_val_ - red_val_ );
norm_val = norm_val < 0.0 ? 0.0 : norm_val;
norm_val = norm_val > 1.0 ? 1.0 : norm_val;
const int B = 0;
const int G = norm_val * 255;
const int R = ( 1.0 - norm_val ) * 255;
return cv::Scalar( B, G, R );
}
cv::Scalar ColorgenLinear::operator()( const float val ) const
{
float norm_val = ( val - val1_ ) / ( val2_ - val1_ );
const int B = color1_.val[0] * (1.0f-norm_val) + color2_.val[0]*norm_val;
const int G = color1_.val[1] * (1.0f-norm_val) + color2_.val[1]*norm_val;
const int R = color1_.val[2] * (1.0f-norm_val) + color2_.val[2]*norm_val;
return cv::Scalar( B, G, R );
}