#include "Visualizer.h" #include #include "affdex_small_logo.h" #include Visualizer::Visualizer(): GREEN_COLOR_CLASSIFIERS({ "joy" }), RED_COLOR_CLASSIFIERS({ "anger", "disgust", "sadness", "fear", "contempt" }) { logo_resized = false; logo = cv::imdecode(cv::InputArray(small_logo), CV_LOAD_IMAGE_UNCHANGED); EXPRESSIONS = { "smile", "innerBrowRaise", "browRaise", "browFurrow", "noseWrinkle", "upperLipRaise", "lipCornerDepressor", "chinRaise", "lipPucker", "lipPress", "lipSuck", "mouthOpen", "smirk", "eyeClosure", "attention", "eyeWiden", "cheekRaise", "lidTighten", "dimpler", "lipStretch", "jawDrop" }; EMOTIONS = { "joy", "fear", "disgust", "sadness", "anger", "surprise", "contempt", "valence", "engagement" }; HEAD_ANGLES = { "pitch", "yaw", "roll" }; EMOJIS = std::vector { "relaxed", "smiley", "laughing", "kissing", "disappointed", "rage", "smirk", "wink", "stuckOutTongueWinkingEye", "stuckOutTongue", "flushed", "scream" }; GENDER_MAP = std::map { { affdex::Gender::Male, "male" }, { affdex::Gender::Female, "female" }, { affdex::Gender::Unknown, "unknown" }, }; GLASSES_MAP = std::map { { affdex::Glasses::Yes, "yes" }, { affdex::Glasses::No, "no" } }; AGE_MAP = std::map { { affdex::Age::AGE_UNKNOWN, "unknown"}, { affdex::Age::AGE_UNDER_18, "under 18" }, { affdex::Age::AGE_18_24, "18-24" }, { affdex::Age::AGE_25_34, "25-34" }, { affdex::Age::AGE_35_44, "35-44" }, { affdex::Age::AGE_45_54, "45-54" }, { affdex::Age::AGE_55_64, "55-64" }, { affdex::Age::AGE_65_PLUS, "65 plus" } }; ETHNICITY_MAP = std::map { { affdex::Ethnicity::UNKNOWN, "unknown"}, { affdex::Ethnicity::CAUCASIAN, "caucasian" }, { affdex::Ethnicity::BLACK_AFRICAN, "black african" }, { affdex::Ethnicity::SOUTH_ASIAN, "south asian" }, { affdex::Ethnicity::EAST_ASIAN, "east asian" }, { affdex::Ethnicity::HISPANIC, "hispanic" } }; } void Visualizer::drawFaceMetrics(affdex::Face face, std::vector bounding_box) { cv::Scalar white_color = cv::Scalar(255, 255, 255); //Draw Right side metrics int padding = bounding_box[0].y; //Top left Y drawValues((float *)&face.expressions, EXPRESSIONS, bounding_box[2].x + spacing, padding, white_color, false); padding = bounding_box[2].y; //Top left Y //Draw Head Angles drawHeadOrientation(face.measurements.orientation, bounding_box[0].x - spacing, padding); //Draw Appearance drawAppearance(face.appearance, bounding_box[0].x - spacing, padding); //Draw Left side metrics drawValues((float *)&face.emotions, EMOTIONS, bounding_box[0].x - spacing, padding, white_color, true); } void Visualizer::drawValues(const float * first, const std::vector names, const int x, int &padding, const cv::Scalar clr, const bool align_right) { for (std::string name : names) { drawClassifierOutput(name, (*first), cv::Point(x, padding += spacing), align_right); first++; } } void Visualizer::updateImage(cv::Mat output_img) { img = output_img; if (!logo_resized) { double logo_width = (logo.size().width > img.size().width*0.25 ? img.size().width*0.25 : logo.size().width); double logo_height = ((double)logo_width) * ((double)logo.size().height / logo.size().width); cv::resize(logo, logo, cv::Size(logo_width, logo_height)); logo_resized = true; } cv::Mat roi = img(cv::Rect(img.cols - logo.cols - 10, 10, logo.cols, logo.rows)); overlayImage(logo, roi, cv::Point(0, 0)); } void Visualizer::drawPoints(affdex::VecFeaturePoint points) { for (auto& point : points) //Draw face feature points. { cv::circle(img, cv::Point(point.x, point.y), 2.0f, cv::Scalar(255, 255, 255)); } } void Visualizer::drawBoundingBox(cv::Point2f top_left, cv::Point2f bottom_right, float valence) { //Draw bounding box const ColorgenRedGreen valence_color_generator( -100, 100 ); cv::rectangle( img, top_left, bottom_right, valence_color_generator(valence), 3); } /** @brief DrawText prints text on screen either right or left justified at the anchor location (loc) * @param output_img -- Image we are plotting on * @param name -- Name of the classifier * @param value -- Value we are trying to display * @param loc -- Exact location. When aligh_right is (true/false) this should be the (upper-right, upper-left) * @param align_right -- Whether to right or left justify the text * @param color -- Color */ void Visualizer::drawText(const std::string& name, const std::string& value, const cv::Point2f loc, bool align_right, cv::Scalar color) { const int block_width = 8; const int margin = 2; const int block_size = 10; const int max_blocks = 100/block_size; cv::Point2f display_loc = loc; const std::string label = name+": "; if( align_right ) { display_loc.x -= (margin+block_width) * max_blocks; int baseline=0; cv::Size txtSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5f, 5,&baseline); display_loc.x -= txtSize.width; } cv::putText(img, label+value, display_loc, cv::FONT_HERSHEY_SIMPLEX, 0.5f, color, 1); } /** @brief DrawClassifierOutput handles choosing between equalizer or text as well as defining the colors * @param name -- Name of the classifier * @param value -- Value we are trying to display * @param loc -- Exact location. When aligh_right is (true/false) this should be the (upper-right, upper-left) * @param align_right -- Whether to right or left justify the text */ void Visualizer::drawClassifierOutput(const std::string& classifier, const float value, const cv::Point2f& loc, bool align_right) { static const ColorgenLinear white_yellow_generator( 0, 100, cv::Scalar(255,255,255), cv::Scalar(0, 255, 255)); static const ColorgenRedGreen valence_color_generator( -100, 100 ); // Determine the display color cv::Scalar color = cv::Scalar(255, 255, 255); if( classifier == "valence") { color = valence_color_generator( value ); } else if( RED_COLOR_CLASSIFIERS.count(classifier) ) { color = cv::Scalar(0, 0, 255); } else if( GREEN_COLOR_CLASSIFIERS.count(classifier) ) { color = cv::Scalar(0, 255, 0); } float equalizer_magnitude = value; if( classifier == "valence" ) { equalizer_magnitude = std::fabs(value); } drawEqualizer(classifier, equalizer_magnitude, loc, align_right, color ); } void Visualizer::drawEqualizer(const std::string& name, const float value, const cv::Point2f& loc, bool align_right, cv::Scalar color) { const int block_width = 8; const int block_height = 10; const int margin = 2; const int block_size = 10; const int max_blocks = 100/block_size; int blocks = round(value / block_size); int i = loc.x, j = loc.y - 10; cv::Point2f display_loc = loc; const std::string label = align_right? name+": " : " :"+name; for (int x = 0 ; x < (100/block_size) ; x++) { cv::Scalar scalar_clr = color; float alpha = 0.8; const int ii = (std::max)( float(i), 0.0f); const int jj = (std::max)( float(j), 0.0f); const int width = (std::min)(float(block_width), float(img.size().width-ii)); const int height = (std::min)(float(block_height), float(img.size().height-jj)); if (height < 0 || width < 0) continue; cv::Mat roi = img(cv::Rect(ii, jj, width, height)); if (x >= blocks) { alpha = 0.3; scalar_clr = cv::Scalar(186, 186, 186); } cv::Mat color(roi.size(), CV_8UC3, scalar_clr); cv::addWeighted(color, alpha, roi, 1.0 - alpha , 0.0, roi); i += align_right? -(margin+block_width):(margin+block_width); } display_loc.x += align_right? -(margin+block_width) * max_blocks : (margin+block_width) * max_blocks; if( align_right ) { int baseline=0; cv::Size txtSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5f, 5,&baseline); display_loc.x -= txtSize.width; } cv::putText(img, label, display_loc, cv::FONT_HERSHEY_SIMPLEX, 0.5f, cv::Scalar(50,50,50), 5); cv::putText(img, label, display_loc, cv::FONT_HERSHEY_SIMPLEX, 0.5f, cv::Scalar(255, 255, 255), 1); } void Visualizer::drawHeadOrientation(affdex::Orientation headAngles, const int x, int &padding, bool align_right, cv::Scalar color) { std::string valueStr = boost::str(boost::format("%3.1f") % headAngles.pitch); drawText("pitch", valueStr, cv::Point(x, padding += spacing), align_right, color ); valueStr = boost::str(boost::format("%3.1f") % headAngles.yaw); drawText("yaw", valueStr, cv::Point(x, padding += spacing), align_right, color ); valueStr = boost::str(boost::format("%3.1f") % headAngles.roll); drawText("roll", valueStr, cv::Point(x, padding += spacing), align_right, color ); } void Visualizer::drawAppearance(affdex::Appearance appearance, const int x, int &padding, bool align_right, cv::Scalar color) { drawText("gender", GENDER_MAP[appearance.gender], cv::Point(x, padding += spacing), align_right, color ); drawText("age", AGE_MAP[appearance.age], cv::Point(x, padding += spacing), align_right, color ); drawText("ethnicity", ETHNICITY_MAP[appearance.ethnicity], cv::Point(x, padding += spacing), align_right, color ); } void Visualizer::showImage() { cv::imshow("analyze video", img); cv::waitKey(5); } void Visualizer::overlayImage(const cv::Mat &foreground, cv::Mat &background, cv::Point2i location) { // start at the row indicated by location, or at row 0 if location.y is negative. for(int y = (std::max)(location.y , 0); y < background.rows; ++y) { int fY = y - location.y; // because of the translation // we are done of we have processed all rows of the foreground image. if(fY >= foreground.rows) break; // start at the column indicated by location, // or at column 0 if location.x is negative. for(int x = (std::max)(location.x, 0); x < background.cols; ++x) { int fX = x - location.x; // because of the translation. // we are done with this row if the column is outside of the foreground image. if(fX >= foreground.cols) break; // determine the opacity of the foregrond pixel, using its fourth (alpha) channel. double opacity = ((double)foreground.data[fY * foreground.step + fX * foreground.channels() + (foreground.channels()-1)]) / 255.; // and now combine the background and foreground pixel, using the opacity, // but only if opacity > 0. for(int c = 0; opacity > 0 && c < background.channels(); ++c) { unsigned char foregroundPx = foreground.data[fY * foreground.step + fX * foreground.channels() + c]; unsigned char backgroundPx = background.data[y * background.step + x * background.channels() + c]; background.data[y*background.step + background.channels()*x + c] = backgroundPx * (1.-opacity) + foregroundPx * opacity; } } } } cv::Scalar ColorgenRedGreen::operator()( const float val ) const { float norm_val = ( val - red_val_ ) / ( green_val_ - red_val_ ); norm_val = norm_val < 0.0 ? 0.0 : norm_val; norm_val = norm_val > 1.0 ? 1.0 : norm_val; const int B = 0; const int G = norm_val * 255; const int R = ( 1.0 - norm_val ) * 255; return cv::Scalar( B, G, R ); } cv::Scalar ColorgenLinear::operator()( const float val ) const { float norm_val = ( val - val1_ ) / ( val2_ - val1_ ); const int B = color1_.val[0] * (1.0f-norm_val) + color2_.val[0]*norm_val; const int G = color1_.val[1] * (1.0f-norm_val) + color2_.val[1]*norm_val; const int R = color1_.val[2] * (1.0f-norm_val) + color2_.val[2]*norm_val; return cv::Scalar( B, G, R ); }