From b6e1fd15b6957db71c2b7de241ec76b6870bf756 Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Thu, 3 Sep 2015 15:50:04 -0400 Subject: [PATCH 1/5] Changes to opencv-webcam-demo app following SDK 2.0 --- opencv-webcam-demo/opencv-webcam-demo.cpp | 81 +++++++++++++++-------- 1 file changed, 55 insertions(+), 26 deletions(-) diff --git a/opencv-webcam-demo/opencv-webcam-demo.cpp b/opencv-webcam-demo/opencv-webcam-demo.cpp index 8d7d55c..c5be8e6 100644 --- a/opencv-webcam-demo/opencv-webcam-demo.cpp +++ b/opencv-webcam-demo/opencv-webcam-demo.cpp @@ -25,39 +25,72 @@ class PlottingImageListener : public ImageListener public: void onImageResults(std::map faces, Frame image) { + const int spacing = 10; + const int left_margin = 30; + const int font = cv::FONT_HERSHEY_COMPLEX_SMALL; + float font_size = 0.5f; + cv::Scalar clr = cv::Scalar(0, 0, 255); + cv::Scalar header_clr = cv::Scalar(255, 0, 0); shared_ptr imgdata = image.getBGRByteArray(); - cv::Mat img = cv::Mat(image.getHeight(), image.getWidth(), CV_8UC3, imgdata.get()); - for (int i = 0; i < faces.size(); i++) + cv::Mat mImg = cv::Mat(image.getHeight(), image.getWidth(), CV_8UC3, imgdata.get()); + + for (unsigned int i = 0; i < faces.size(); i++) { Face f = faces[i]; - float smile_score = f.getSmileScore(); - int n = f.getFeaturePointCount(); - VecFeaturePoint points = f.getFeaturePoints(); - for (auto& point : points ) //Draw face feature points. + VecFeaturePoint points = f.featurePoints; + for (auto& point : points) //Draw face feature points. { - cv::circle(img, cv::Point(point.x, point.y), 1.0f, cv::Scalar(0, 0, 255)); + cv::circle(mImg, cv::Point(point.x, point.y), 1.0f, cv::Scalar(0, 0, 255)); } + Orientation headAngles = f.measurements.orientation; + std::string strAngles = "Pitch: " + std::to_string(headAngles.pitch) + + " Yaw: " + std::to_string(headAngles.yaw) + + " Roll: " + std::to_string(headAngles.roll) + + " InterOcularDist: " + std::to_string(f.measurements.interocularDistance); //Output the results of the different classifiers. - cv::putText(img, "Smile: "+ std::to_string(f.getSmileScore()), cv::Point(30, 30), cv::FONT_HERSHEY_COMPLEX, 0.5f, cv::Scalar(0, 0, 255)); - cv::putText(img, "BrowFurrow: " + std::to_string(f.getBrowFurrowScore()), cv::Point(30, 50), cv::FONT_HERSHEY_COMPLEX, 0.5f, cv::Scalar(0, 0, 255)); - cv::putText(img, "BrowRaise: " + std::to_string(f.getBrowRaiseScore()), cv::Point(30, 70), cv::FONT_HERSHEY_COMPLEX, 0.5f, cv::Scalar(0, 0, 255)); - cv::putText(img, "LipCornerDepressor: " + std::to_string(f.getLipCornerDepressorScore()), cv::Point(30, 90), cv::FONT_HERSHEY_COMPLEX, 0.5f, cv::Scalar(0, 0, 255)); - cv::putText(img, "Engagement: " + std::to_string(f.getEngagementScore()), cv::Point(30, 110), cv::FONT_HERSHEY_COMPLEX, 0.5f, cv::Scalar(0, 0, 255)); - cv::putText(img, "Valence: " + std::to_string(f.getValenceScore()), cv::Point(30, 130), cv::FONT_HERSHEY_COMPLEX, 0.5f, cv::Scalar(0, 0, 255)); + int padding = 10; + std::vector expressions{ "smile", "innerBrowRaise", "browRaise", "browFurrow", "noseWrinkle", + "upperLipRaise", "lipCornerDepressor", "chinRaise", "lipPucker", "lipPress", + "lipSuck", "mouthOpen", "smirk", "eyeClosure", "attention" }; - //Calculate the processing framerate, output both the processing + capture framerate - if (process_last_timestamp >= 0.0f) + std::vector emotions{ "joy", "fear", "disgust", "sadness", "anger", "surprise", "contempt", "valence", "engagement" }; + + cv::putText(mImg, "MEASUREMENTS", cv::Point(left_margin, padding += spacing), font, font_size, header_clr); + + cv::putText(mImg, strAngles, cv::Point(left_margin, padding += spacing), font, font_size, clr); + + cv::putText(mImg, "EXPRESSIONS", cv::Point(left_margin, padding += (spacing * 2)), font, font_size, header_clr); + + float * values = (float *)&f.expressions; + for (string expression : expressions) { - process_fps = 1.0f / (image.getTimestamp() - process_last_timestamp); - cv::putText(img, "capture fps: " + std::to_string(capture_fps), cv::Point(img.cols - 200, 30), cv::FONT_HERSHEY_COMPLEX, 0.5f, cv::Scalar(0, 0, 255)); - cv::putText(img, "process fps: " + std::to_string(process_fps), cv::Point(img.cols - 200, 50), cv::FONT_HERSHEY_COMPLEX, 0.5f, cv::Scalar(0, 0, 255)); + cv::putText(mImg, expression + ": " + std::to_string(int(*values)), cv::Point(left_margin, padding += spacing), font, font_size, clr); + values++; } + + cv::putText(mImg, "EMOTIONS", cv::Point(left_margin, padding += (spacing * 2)), font, font_size, header_clr); + + values = (float *)&f.emotions; + + for (string emotion : emotions) + { + cv::putText(mImg, emotion + ": " + std::to_string(int(*values)), cv::Point(left_margin, padding += spacing), font, font_size, clr); + values++; + } + std::cerr << "Timestamp: " << image.getTimestamp() + << "," << image.getWidth() + << "x" << image.getHeight() + << " cfps: " << capture_fps + << " pnts: " << points.size() << endl; process_last_timestamp = image.getTimestamp(); + } - cv::imshow("analyze-image", img); - + cv::putText(mImg, "capture fps: " + std::to_string(int(capture_fps)), cv::Point(mImg.cols - 110, mImg.rows - left_margin - spacing), font, font_size, clr); + + cv::imshow("analyze-image", mImg); + cv::waitKey(30); }; @@ -91,12 +124,8 @@ int main(int argsc, char ** argsv) } //Initialize detectors - frameDetector.setDetectSmile(true); - frameDetector.setDetectBrowFurrow(true); - frameDetector.setDetectBrowRaise(true); - frameDetector.setDetectLipCornerDepressor(true); - frameDetector.setDetectEngagement(true); - frameDetector.setDetectValence(true); + frameDetector.setDetectAllEmotions(true); + frameDetector.setDetectAllExpressions(true); frameDetector.setClassifierPath(AFFDEX_DATA_DIR); frameDetector.setLicensePath(AFFDEX_LICENSE_FILE); frameDetector.setImageListener(listenPtr.get()); From e4ba8321e05038061f3429099cb095d30be74795 Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Thu, 3 Sep 2015 15:52:25 -0400 Subject: [PATCH 2/5] Fix documentation --- AffdexMe/README.md | 2 +- README.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AffdexMe/README.md b/AffdexMe/README.md index 70b6497..f300403 100644 --- a/AffdexMe/README.md +++ b/AffdexMe/README.md @@ -18,7 +18,7 @@ This application runs on Windows 7.0, 8.0 and 8.1 * Visual Studio 2013 -* To download and install the Windows SDK from Affectiva By default, the Windows SDK is installed to the following location: C:\Program Files (x86)\Affectiva\Affdex SDK +* To download and install the Windows SDK (32-bit) from Affectiva By default, the Windows SDK is installed to the following location: C:\Program Files (x86)\Affectiva\Affdex SDK If you have installed the SDK to a location other than the default, you will have to modify the following String constants located in the MainWindow.xaml.cs file: diff --git a/README.md b/README.md index 7ff921d..93b9cbb 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ #Sample Apps for Affdex SDK for Windows -Welcome to our repository on GitHub! Here you will find example code to get you started with our Affdex SDK for Windows and begin emotion-enabling you own app! +Welcome to our repository on GitHub! Here you will find example code to get you started with our Affdex SDK 2.0 for Windows and begin emotion-enabling you own app! OpenCV-webcam-demo ------------------ *Dependencies* +- Affdex SDK 2.0 (32 bit) - OpenCV for Windows 2.4.9: http://sourceforge.net/projects/opencvlibrary/files/opencv-win/2.4.9/ - Visual Studio 2013 or higher From 383bb912e733226ad109a5a12353c388c4412cd8 Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Thu, 3 Sep 2015 16:31:43 -0400 Subject: [PATCH 3/5] AffdexMe upgraded to include a new selection ui as well as enable / disable all expressions and emotions --- AffdexMe/AffdexMe.csproj | 68 +++- AffdexMe/MainWindow.xaml | 76 ++-- AffdexMe/MainWindow.xaml.cs | 440 +++++++++++++++-------- AffdexMe/Properties/Settings.Designer.cs | 68 +++- AffdexMe/Properties/Settings.settings | 26 +- 5 files changed, 461 insertions(+), 217 deletions(-) diff --git a/AffdexMe/AffdexMe.csproj b/AffdexMe/AffdexMe.csproj index a176850..dd0b15f 100644 --- a/AffdexMe/AffdexMe.csproj +++ b/AffdexMe/AffdexMe.csproj @@ -37,7 +37,7 @@ 4 - + False c:\Program Files (x86)\Affectiva\Affdex SDK\bin\release\Affdex.dll @@ -55,8 +55,14 @@ + + $(ProjectDir)packages\WpfAnimatedGif.1.4.13\lib\net\WpfAnimatedGif.dll + + + + MSBuild:Compile Designer @@ -68,6 +74,10 @@ $(ProjectDir)MainWindow.xaml Code + + MSBuild:Compile + Designer + @@ -102,9 +112,59 @@ - - PreserveNewest - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/AffdexMe/MainWindow.xaml b/AffdexMe/MainWindow.xaml index 3412242..790e98b 100644 --- a/AffdexMe/MainWindow.xaml +++ b/AffdexMe/MainWindow.xaml @@ -28,7 +28,7 @@ - + @@ -81,7 +81,7 @@ - + @@ -103,7 +103,8 @@ - + @@ -127,13 +128,16 @@ - + Height="auto" Source="AffectivaLogo1.png" Visibility="Visible"/> + + + + @@ -158,51 +162,51 @@ - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -224,7 +228,7 @@ + Height="auto" Source="AffectivaLogo1.png" Margin="0,0,0,-34" Grid.RowSpan="2"/> @@ -234,11 +238,13 @@ -