affdex-sdk-cpp-samples/common/PlottingImageListener.hpp

288 lines
9.7 KiB
C++
Raw Normal View History

Squashed commit of the following: commit 45c5b72d6c294fe1a4cd90f8090442d313886fc9 Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Wed Feb 3 07:17:09 2016 -0500 Remove affdex me .. fix dll references commit 98bfdbc3302c1ba8226a4151826c541d8d250aa9 Author: Abdelrahman Mahmoud <ahamino@gmail.com> Date: Wed Feb 3 06:57:59 2016 -0500 Update README.md commit fd9dedbdc7daeeacc65ef1df8b79447e5e22cc64 Author: Abdelrahman Mahmoud <ahamino@gmail.com> Date: Tue Feb 2 15:43:00 2016 -0500 Update README.md update appveyor badge commit ccbd9770da316c35de75b47d749e14bd551acc2a Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Tue Feb 2 15:36:11 2016 -0500 Update appveyor badge commit b08547f1dd77548f24140e709a8574d23c8aa7db Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Tue Feb 2 15:32:03 2016 -0500 Add appveyor badge commit e0a2b36ca9fb0c828bf54c7326aa59a1fa6d1079 Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Tue Feb 2 15:11:50 2016 -0500 remove nuget references commit b5561f7353381106ad5b133d3bc73d49d36bcbc0 Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Tue Feb 2 15:01:15 2016 -0500 remove nuget commit ec3d93bb38e5390f176de23dd249b30e5420f3fb Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Tue Feb 2 14:09:28 2016 -0500 More changes to get nuget working commit ea9662b583d08fa9bae3b30cb2d909d288911b2e Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Tue Feb 2 12:53:20 2016 -0500 Change the opencv version commit 56ffaebfb3c3cdc1610f1ad2c67a47010d0e3dbf Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Tue Feb 2 12:29:22 2016 -0500 Commit changes to the nuget configuration commit 3e6aa2c2dc4a8cb97dd9999154b8c6fb5471c351 Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Tue Feb 2 12:12:03 2016 -0500 add nuget configuration commit e69ea5e078b42a286f0ca27b09b922178e376cd1 Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Mon Feb 1 17:02:38 2016 -0500 Remove unecessary packages from nuget commit 3f98b04af745b880be3f4be5d4e7d7a6174f2453 Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Mon Dec 21 18:22:45 2015 -0500 Update the documentation commit a92352bea891e553fb8f8da61afda3fa255ec46f Author: Abdelrahman Mahmoud <mahmoud@affectiva.com> Date: Mon Dec 21 17:54:55 2015 -0500 Add video demo 2.1
2016-02-03 12:20:08 +00:00
#pragma once
#include <iostream>
#include <memory>
#include <chrono>
#include <thread>
#include <mutex>
#include <fstream>
#include <map>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <boost/filesystem.hpp>
#include <boost/timer/timer.hpp>
#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include "ImageListener.h"
using namespace affdex;
class PlottingImageListener : public ImageListener
{
std::mutex mMutex;
std::deque<std::pair<Frame, std::map<FaceId, Face> > > mDataArray;
double mCaptureLastTS;
double mCaptureFPS;
double mProcessLastTS;
double mProcessFPS;
std::ofstream &fStream;
std::chrono::time_point<std::chrono::system_clock> mStartT;
const bool mDrawDisplay;
const int spacing = 10;
const float font_size = 0.5f;
const int font = cv::FONT_HERSHEY_COMPLEX_SMALL;
std::vector<std::string> expressions;
std::vector<std::string> emotions;
std::vector<std::string> headAngles;
std::map<affdex::Glasses, std::string> glassesMap;
std::map<affdex::Gender, std::string> genderMap;
public:
PlottingImageListener(std::ofstream &csv, const bool draw_display)
: fStream(csv), mDrawDisplay(draw_display), mStartT(std::chrono::system_clock::now()),
mCaptureLastTS(-1.0f), mCaptureFPS(-1.0f),
mProcessLastTS(-1.0f), mProcessFPS(-1.0f)
{
expressions = {
"smile", "innerBrowRaise", "browRaise", "browFurrow", "noseWrinkle",
"upperLipRaise", "lipCornerDepressor", "chinRaise", "lipPucker", "lipPress",
"lipSuck", "mouthOpen", "smirk", "eyeClosure", "attention"
};
emotions = {
"joy", "fear", "disgust", "sadness", "anger",
"surprise", "contempt", "valence", "engagement"
};
headAngles = { "pitch", "yaw", "roll" };
genderMap = std::map<affdex::Gender, std::string> {
{ affdex::Gender::Male, "male" },
{ affdex::Gender::Female, "female" },
{ affdex::Gender::Unknown, "unknown" },
};
glassesMap = std::map<affdex::Glasses, std::string> {
{ affdex::Glasses::Yes, "glasses" },
{ affdex::Glasses::No, "no glasses" }
};
fStream << "TimeStamp,faceId,interocularDistance,glasses,gender,";
for (std::string angle : headAngles) fStream << angle << ",";
for (std::string emotion : emotions) fStream << emotion << ",";
for (std::string expression : expressions) fStream << expression << ",";
fStream << std::endl;
fStream.precision(4);
fStream << std::fixed;
}
FeaturePoint minPoint(VecFeaturePoint points)
{
VecFeaturePoint::iterator it = points.begin();
FeaturePoint ret = *it;
for (; it != points.end(); it++)
{
if (it->x < ret.x) ret.x = it->x;
if (it->y < ret.y) ret.y = it->y;
}
return ret;
};
FeaturePoint maxPoint( VecFeaturePoint points)
{
VecFeaturePoint::iterator it = points.begin();
FeaturePoint ret = *it;
for (; it != points.end(); it++)
{
if (it->x > ret.x) ret.x = it->x;
if (it->y > ret.y) ret.y = it->y;
}
return ret;
};
double getProcessingFrameRate()
{
std::lock_guard<std::mutex> lg(mMutex);
return mProcessFPS;
}
double getCaptureFrameRate()
{
std::lock_guard<std::mutex> lg(mMutex);
return mCaptureFPS;
}
int getDataSize()
{
std::lock_guard<std::mutex> lg(mMutex);
return mDataArray.size();
}
std::pair<Frame, std::map<FaceId, Face>> getData()
{
std::lock_guard<std::mutex> lg(mMutex);
std::pair<Frame, std::map<FaceId, Face>> dpoint = mDataArray.front();
mDataArray.pop_front();
return dpoint;
}
void onImageResults(std::map<FaceId, Face> faces, Frame image) override
{
std::lock_guard<std::mutex> lg(mMutex);
mDataArray.push_back(std::pair<Frame, std::map<FaceId, Face>> (image, faces) );
std::chrono::time_point<std::chrono::system_clock> now = std::chrono::system_clock::now();
std::chrono::milliseconds milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(now - mStartT);
double seconds = milliseconds.count() / 1000.f;
mProcessFPS = 1.0f / (seconds - mProcessLastTS);
mProcessLastTS = seconds;
};
void onImageCapture(Frame image) override
{
std::lock_guard<std::mutex> lg(mMutex);
mCaptureFPS = 1.0f / (image.getTimestamp() - mCaptureLastTS);
mCaptureLastTS = image.getTimestamp();
};
void outputToFile(const std::map<FaceId, Face> faces, const double timeStamp)
{
if (faces.empty())
{
fStream << timeStamp << "nan,nan,no glasses,unknown,";
for (std::string angle : headAngles) fStream << "nan,";
for (std::string emotion : emotions) fStream << "nan,";
for (std::string expression : expressions) fStream << "nan,";
fStream << std::endl;
}
for (auto & face_id_pair : faces)
{
Face f = face_id_pair.second;
fStream << timeStamp << ","
<< f.id << ","
<< f.measurements.interocularDistance << ","
<< glassesMap[f.appearance.glasses] << ","
<< genderMap[f.appearance.gender] << ",";
float *values = (float *)&f.measurements.orientation;
for (std::string angle : headAngles)
{
fStream << (*values) << ",";
values++;
}
values = (float *)&f.emotions;
for (std::string emotion : emotions)
{
fStream << (*values) << ",";
values++;
}
values = (float *)&f.expressions;
for (std::string expression : expressions)
{
fStream << (*values) << ",";
values++;
}
fStream << std::endl;
}
}
void drawValues(const float * first, const std::vector<std::string> names,
const int x, int &padding, const cv::Scalar clr,
cv::Mat img)
{
for (std::string name : names)
{
if ((*first) > 5.0f)
{
char m[50];
sprintf(m, "%s: %3.2f", name.c_str(), (*first));
cv::putText(img, m, cv::Point(x, padding += spacing), font, font_size, clr);
}
first++;
}
}
void draw(const std::map<FaceId, Face> faces, Frame image)
{
std::shared_ptr<byte> imgdata = image.getBGRByteArray();
cv::Mat img = cv::Mat(image.getHeight(), image.getWidth(), CV_8UC3, imgdata.get());
const int left_margin = 30;
cv::Scalar clr = cv::Scalar(255, 255, 255);
cv::Scalar header_clr = cv::Scalar(255, 0, 0);
for (auto & face_id_pair : faces)
{
Face f = face_id_pair.second;
VecFeaturePoint points = f.featurePoints;
for (auto& point : points) //Draw face feature points.
{
cv::circle(img, cv::Point(point.x, point.y), 2.0f, cv::Scalar(0, 0, 255));
}
FeaturePoint tl = minPoint(points);
FeaturePoint br = maxPoint(points);
//Output the results of the different classifiers.
int padding = tl.y + 10;
cv::putText(img, "APPEARANCE", cv::Point(br.x, padding += (spacing * 2)), font, font_size, header_clr);
cv::putText(img, genderMap[f.appearance.gender], cv::Point(br.x, padding += spacing), font, font_size, clr);
cv::putText(img, glassesMap[f.appearance.glasses], cv::Point(br.x, padding += spacing), font, font_size, clr);
Orientation headAngles = f.measurements.orientation;
char strAngles[100];
sprintf(strAngles, "Pitch: %3.2f Yaw: %3.2f Roll: %3.2f Interocular: %3.2f",
headAngles.pitch, headAngles.yaw, headAngles.roll, f.measurements.interocularDistance);
char fId[10];
sprintf(fId, "ID: %i", f.id);
cv::putText(img, fId, cv::Point(br.x, padding += spacing), font, font_size, clr);
cv::putText(img, "MEASUREMENTS", cv::Point(br.x, padding += (spacing * 2)), font, font_size, header_clr);
cv::putText(img, strAngles, cv::Point(br.x, padding += spacing), font, font_size, clr);
cv::putText(img, "EXPRESSIONS", cv::Point(br.x, padding += (spacing * 2)), font, font_size, header_clr);
drawValues((float *)&f.expressions, expressions, br.x, padding, clr, img);
cv::putText(img, "EMOTIONS", cv::Point(br.x, padding += (spacing * 2)), font, font_size, header_clr);
drawValues((float *)&f.emotions, emotions, br.x, padding, clr, img);
}
char fps_str[50];
sprintf(fps_str, "capture fps: %2.0f", mCaptureFPS);
cv::putText(img, fps_str, cv::Point(img.cols - 110, img.rows - left_margin - spacing), font, font_size, clr);
sprintf(fps_str, "process fps: %2.0f", mProcessFPS);
cv::putText(img, fps_str, cv::Point(img.cols - 110, img.rows - left_margin), font, font_size, clr);
cv::imshow("analyze video", img);
cv::waitKey(5);
}
};