Making sure stuff compiles after the merge.

gui/OpenFaceOffline/MainWindow.xaml.cs

@@ -49,6 +49,7 @@ using CppInterop.LandmarkDetector;
 using CameraInterop;
 using FaceAnalyser_Interop;
 using GazeAnalyser_Interop;
+using MediaReader;
 using System.Globalization;
 using Microsoft.WindowsAPICodePack.Dialogs;
 
@@ -85,6 +86,7 @@ namespace OpenFaceOffline
 
         // Some members for displaying the results
         private Capture capture;
+        private ImageReader image_reader;
         private WriteableBitmap latest_img;
         private WriteableBitmap latest_aligned_face;
         private WriteableBitmap latest_HOG_descriptor;

lib/local/CppInerop/CppInerop.vcxproj

@@ -116,7 +116,7 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
       <PreprocessorDefinitions>WIN32;_DEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>
@@ -130,7 +130,7 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
       <PreprocessorDefinitions>_DEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>
@@ -146,7 +146,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>
@@ -164,7 +164,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <PreprocessorDefinitions>NDEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>
@@ -181,6 +181,7 @@
     <ClInclude Include="CameraInterop.h" />
     <ClInclude Include="FaceAnalyserInterop.h" />
     <ClInclude Include="GazeAnalyserInterop.h" />
+    <ClInclude Include="ImageReader.h" />
     <ClInclude Include="LandmarkDetectorInterop.h" />
     <ClInclude Include="OpenCVWrappers.h" />
   </ItemGroup>
@@ -200,6 +201,9 @@
     <ProjectReference Include="..\LandmarkDetector\LandmarkDetector.vcxproj">
       <Project>{bdc1d107-de17-4705-8e7b-cdde8bfb2bf8}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\Utilities\Utilities.vcxproj">
+      <Project>{8e741ea2-9386-4cf2-815e-6f9b08991eac}</Project>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
     <Reference Include="PresentationCore" />

lib/local/CppInerop/CppInerop.vcxproj.filters

@@ -38,5 +38,8 @@
     <ClInclude Include="GazeAnalyserInterop.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="ImageReader.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

lib/local/CppInerop/CppInterop.cpp

@@ -38,3 +38,4 @@
 #include "GazeAnalyserInterop.h"
 #include "OpenCVWrappers.h"
 #include "CameraInterop.h"
+#include "ImageReader.h"

lib/local/CppInerop/FaceAnalyserInterop.h

@@ -61,6 +61,7 @@
 #include <OpenCVWrappers.h>
 #include <Face_utils.h>
 #include <FaceAnalyser.h>
+#include <VisualizationUtils.h>
 
 // Boost stuff
 #include <filesystem.hpp>
@@ -187,7 +188,7 @@ public:
 		face_analyser->PostprocessOutputFile(msclr::interop::marshal_as<std::string>(file));
 	}
 
-	void AddNextFrame(OpenCVWrappers::RawImage^ frame, List<System::Tuple<double, double>^>^ landmarks, bool success, bool online) {
+	void AddNextFrame(OpenCVWrappers::RawImage^ frame, List<System::Tuple<double, double>^>^ landmarks, bool success, bool online, bool vis_hog) {
 			
 		// Construct an OpenCV matric from the landmarks
 		cv::Mat_<double> landmarks_mat(landmarks->Count * 2, 1, 0.0);
@@ -209,7 +210,7 @@ public:
 
 		if(vis_hog)
 		{
-			*visualisation = face_analyser->GetLatestHOGDescriptorVisualisation();
+			Utilities::Visualise_FHOG(*hog_features, *num_rows, *num_cols, *visualisation);
 		}
 
 	}
@@ -227,10 +228,10 @@ public:
 			landmarks_mat.at<double>(i + landmarks->Count, 0) = landmarks[i]->Item2;
 		}
 
-		auto aus = face_analyser->PredictStaticAUs(frame->Mat, landmarks_mat, false);
+		face_analyser->PredictStaticAUsAndComputeFeatures(frame->Mat, landmarks_mat);
 
-		auto AU_predictions_intensity = aus.first;
+		auto AU_predictions_intensity = face_analyser->GetCurrentAUsReg();
-		auto AU_predictions_occurence = aus.second;
+		auto AU_predictions_occurence = face_analyser->GetCurrentAUsClass();
 
 		auto au_intensities = gcnew Dictionary<System::String^, double>();
 		auto au_occurences = gcnew Dictionary<System::String^, double>();

lib/local/CppInerop/ImageReader.h

@@ -0,0 +1,191 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
+// all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace: an open source facial behavior analysis toolkit
+//       Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency
+//       in IEEE Winter Conference on Applications of Computer Vision, 2016  
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+//       Constrained Local Neural Fields for robust facial landmark detection in the wild.
+//       Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. 
+//       in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013.    
+//
+///////////////////////////////////////////////////////////////////////////////
+
+// Camera_Interop.h
+
+#pragma once
+
+#pragma unmanaged
+
+// Include all the unmanaged things we need.
+
+#include <opencv2/core/core.hpp>
+#include "opencv2/objdetect.hpp"
+#include "opencv2/calib3d.hpp"
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <vector>
+#include <set>
+
+#include <OpenCVWrappers.h>
+
+#include "ImageCapture.h"
+
+#pragma managed
+
+#include <msclr\marshal.h>
+#include <msclr\marshal_cppstd.h>
+
+namespace MediaReader {
+
+	public ref class ReadingFailedException : System::Exception
+	{
+	public:
+
+		ReadingFailedException(System::String^ message) : Exception(message) {}
+	};
+
+	public ref class ImageReader
+	{
+	private:
+
+		// OpenCV based video capture for reading from files
+		Utilities::ImageCapture* m_image_capture;
+
+		OpenCVWrappers::RawImage^ m_rgb_frame;
+		OpenCVWrappers::RawImage^ m_gray_frame;
+
+		bool* m_is_opened;
+
+	public:
+
+		// Can provide a directory, or a list of files
+		ImageReader(System::String^ image_directory)
+		{
+			m_image_capture = new Utilities::ImageCapture();
+
+			std::string image_dir_std = msclr::interop::marshal_as<std::string>(image_directory);
+
+			*m_is_opened = m_image_capture->OpenDirectory(image_dir_std);
+
+			if (*m_is_opened)
+			{
+				throw gcnew ReadingFailedException("Failed to open a directory or an image");
+			}
+		}
+		// Can provide a directory, or a list of files
+		ImageReader(System::Collections::Generic::List<System::String^>^ image_files)
+		{
+			m_image_capture = new Utilities::ImageCapture();
+
+			std::vector<std::string> image_files_std;
+
+			for (size_t i = 0; i < image_files->Count; ++i)
+			{
+				std::string image_file = msclr::interop::marshal_as<std::string>(image_files[i]);
+				image_files_std.push_back(image_file);
+			}
+
+			bool success = m_image_capture->OpenImageFiles(image_files_std);
+
+			if (!success)
+			{
+				throw gcnew ReadingFailedException("Failed to open a directory or an image");
+			}
+
+		}
+
+		OpenCVWrappers::RawImage^ GetNextImage()
+		{
+			cv::Mat next_image = m_image_capture->GetNextImage();
+
+			if (m_rgb_frame == nullptr)
+			{
+				m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().width, CV_8UC3);
+			}
+
+			next_image.copyTo(m_rgb_frame->Mat);
+
+			return m_rgb_frame;
+		}
+
+		double GetProgress()
+		{
+			return m_image_capture->GetProgress();
+		}
+
+		bool isOpened()
+		{
+			return *m_is_opened;
+		}
+
+		OpenCVWrappers::RawImage^ GetCurrentFrameGray() {
+
+			cv::Mat next_gray_image = m_image_capture->GetGrayFrame();
+
+			if (m_gray_frame == nullptr)
+			{
+				m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().width, CV_8UC3);
+			}
+
+			next_gray_image.copyTo(m_gray_frame->Mat);
+
+			return m_gray_frame;
+		}
+
+		// Finalizer. Definitely called before Garbage Collection,
+		// but not automatically called on explicit Dispose().
+		// May be called multiple times.
+		!ImageReader()
+		{
+			// Automatically closes capture object before freeing memory.	
+			if (m_image_capture != nullptr)
+			{
+				delete m_image_capture;
+			}
+
+			if (m_rgb_frame != nullptr)
+			{
+				delete m_rgb_frame;
+			}
+			if (m_gray_frame != nullptr)
+			{
+				delete m_gray_frame;
+			}
+			if (m_is_opened != nullptr)
+			{
+				delete m_gray_frame;
+			}
+
+		}
+
+		// Destructor. Called on explicit Dispose() only.
+		~ImageReader()
+		{
+			this->!ImageReader();
+		}
+	};
+
+}

lib/local/CppInerop/LandmarkDetectorInterop.h

@@ -63,6 +63,7 @@
 
 #include <Face_utils.h>
 #include <FaceAnalyser.h>
+#include <VisualizationUtils.h>
 
 #ifdef __cplusplus_cli
 #undef generic
@@ -325,26 +326,11 @@ namespace CppInterop {
 				return landmarks_3D;
 			}
 
-
-			// Static functions from the LandmarkDetector namespace.
-			void DrawLandmarks(OpenCVWrappers::RawImage^ img, List<System::Windows::Point>^ landmarks) {
-
-				vector<cv::Point> vecLandmarks;
-
-				for(int i = 0; i < landmarks->Count; i++) {
-					System::Windows::Point p = landmarks[i];
-					vecLandmarks.push_back(cv::Point(p.X, p.Y));
-				}
-
-				::LandmarkDetector::DrawLandmarks(img->Mat, vecLandmarks);
-			}
-
-
 			List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ CalculateBox(float fx, float fy, float cx, float cy) {
 
 				cv::Vec6d pose = ::LandmarkDetector::GetPose(*clnf, fx,fy, cx, cy);
 
-				vector<pair<cv::Point2d, cv::Point2d>> vecLines = ::LandmarkDetector::CalculateBox(pose, fx, fy, cx, cy);
+				vector<pair<cv::Point2d, cv::Point2d>> vecLines = ::Utilities::CalculateBox(pose, fx, fy, cx, cy);
 
 				auto lines = gcnew List<System::Tuple<System::Windows::Point,System::Windows::Point>^>();
 
@@ -355,19 +341,6 @@ namespace CppInterop {
 				return lines;
 			}
 
-			void DrawBox(List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ lines, OpenCVWrappers::RawImage^ image, double r, double g, double b, int thickness) {
-				cv::Scalar color = cv::Scalar(r,g,b,1);
-
-				vector<pair<cv::Point, cv::Point>> vecLines;
-
-				for(int i = 0; i < lines->Count; i++) {
-					System::Tuple<System::Windows::Point, System::Windows::Point>^ points = lines[i];
-					vecLines.push_back(pair<cv::Point, cv::Point>(cv::Point(points->Item1.X, points->Item1.Y), cv::Point(points->Item2.X, points->Item2.Y)));
-				}
-
-				::LandmarkDetector::DrawBox(vecLines, image->Mat, color, thickness);
-			}
-
 			int GetNumPoints()
 			{
 				return clnf->pdm.NumberOfPoints();