moodmeter/common/LoggingImageListener.hpp

#pragma once


#include <iostream>
#include <memory>
#include <chrono>
#include <thread>
#include <mutex>
#include <fstream>
#include <map>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <boost/filesystem.hpp>
#include <boost/timer/timer.hpp>
#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>



#include "ImageListener.h"


using namespace affdex;

/**
 * TOdo: make sure this handles logging to json on onImageResults()
 */
class LoggingImageListener : public ImageListener
{

    std::mutex mMutex;
    std::deque<std::pair<Frame, std::map<FaceId, Face> > > mDataArray;

    double mCaptureLastTS;
    double mCaptureFPS;
    double mProcessLastTS;
    double mProcessFPS;
    std::ofstream &fStream;
    std::chrono::time_point<std::chrono::system_clock> mStartT;
    const bool mDrawDisplay;
    const int spacing = 10;
    const float font_size = 0.5f;
    const int font = cv::FONT_HERSHEY_COMPLEX_SMALL;

    std::vector<std::string> expressions;
    std::vector<std::string> emotions;
    std::vector<std::string> emojis;
    std::vector<std::string> headAngles;

    std::map<affdex::Glasses, std::string> glassesMap;
    std::map<affdex::Gender, std::string> genderMap;
    std::map<affdex::Age, std::string> ageMap;
    std::map<affdex::Ethnicity, std::string> ethnicityMap;

public:


    LoggingImageListener(std::ofstream &csv, const bool draw_display)
        : fStream(csv), mDrawDisplay(draw_display), mStartT(std::chrono::system_clock::now()),
        mCaptureLastTS(-1.0f), mCaptureFPS(-1.0f),
        mProcessLastTS(-1.0f), mProcessFPS(-1.0f)
    {
        expressions = {
            "smile", "innerBrowRaise", "browRaise", "browFurrow", "noseWrinkle",
            "upperLipRaise", "lipCornerDepressor", "chinRaise", "lipPucker", "lipPress",
            "lipSuck", "mouthOpen", "smirk", "eyeClosure", "attention", "eyeWiden", "cheekRaise",
            "lidTighten", "dimpler", "lipStretch", "jawDrop"
        };

        emotions = {
            "joy", "fear", "disgust", "sadness", "anger",
            "surprise", "contempt", "valence", "engagement"
        };

        headAngles = { "pitch", "yaw", "roll" };


        emojis = std::vector<std::string> {
            "relaxed", "smiley", "laughing",
                "kissing", "disappointed",
                "rage", "smirk", "wink",
                "stuckOutTongueWinkingEye", "stuckOutTongue",
                "flushed", "scream"
        };

        genderMap = std::map<affdex::Gender, std::string> {
            { affdex::Gender::Male, "male" },
            { affdex::Gender::Female, "female" },
            { affdex::Gender::Unknown, "unknown" },

        };

        glassesMap = std::map<affdex::Glasses, std::string> {
            { affdex::Glasses::Yes, "yes" },
            { affdex::Glasses::No, "no" }
        };

        ageMap = std::map<affdex::Age, std::string> {
            { affdex::Age::AGE_UNKNOWN, "unknown"},
            { affdex::Age::AGE_UNDER_18, "under 18" },
            { affdex::Age::AGE_18_24, "18-24" },
            { affdex::Age::AGE_25_34, "25-34" },
            { affdex::Age::AGE_35_44, "35-44" },
            { affdex::Age::AGE_45_54, "45-54" },
            { affdex::Age::AGE_55_64, "55-64" },
            { affdex::Age::AGE_65_PLUS, "65 plus" }
        };

        ethnicityMap = std::map<affdex::Ethnicity, std::string> {
            { affdex::Ethnicity::UNKNOWN, "unknown"},
            { affdex::Ethnicity::CAUCASIAN, "caucasian" },
            { affdex::Ethnicity::BLACK_AFRICAN, "black african" },
            { affdex::Ethnicity::SOUTH_ASIAN, "south asian" },
            { affdex::Ethnicity::EAST_ASIAN, "east asian" },
            { affdex::Ethnicity::HISPANIC, "hispanic" }
        };
    }


    void onImageResults(std::map<FaceId, Face> faces, Frame image) override
    {
        std::lock_guard<std::mutex> lg(mMutex);
        mDataArray.push_back(std::pair<Frame, std::map<FaceId, Face>>(image, faces));
        std::chrono::time_point<std::chrono::system_clock> now = std::chrono::system_clock::now();
        std::chrono::milliseconds milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(now - mStartT);
        double seconds = milliseconds.count() / 1000.f;
        mProcessFPS = 1.0f / (seconds - mProcessLastTS);
        mProcessLastTS = seconds;
    };

    void onImageCapture(Frame image) override
    {
        std::lock_guard<std::mutex> lg(mMutex);
        mCaptureFPS = 1.0f / (image.getTimestamp() - mCaptureLastTS);
        mCaptureLastTS = image.getTimestamp();
    };


};