Working towards new interface with GUI

gui/OpenFaceOffline/MainWindow.xaml

@@ -25,9 +25,11 @@
             <MenuItem Header="File">
                 <MenuItem Header="Open video(s)" Click="videoFileOpenClick">
                 </MenuItem>
-                <MenuItem Header="Open image(s)" Click="imageFileOpenClick">
+                <MenuItem Header="Open image directory (as sequence)" Click="imageSequenceFileOpenClick">
                 </MenuItem>
-                <MenuItem Header="Open image sequence" Click="imageSequenceFileOpenClick">
+                <MenuItem Header="Open image(s)" Click="individualImageFilesOpenClick">
+                </MenuItem>
+                <MenuItem Header="Open image directory" Click="imageFileOpenClick">
                 </MenuItem>
             </MenuItem>
             <MenuItem Name="RecordingMenu" Header="Record" >

gui/OpenFaceOffline/MainWindow.xaml.cs

@@ -44,13 +44,12 @@ using Microsoft.Win32;
 
 // Internal libraries
 using OpenCVWrappers;
-using CppInterop;
 using CppInterop.LandmarkDetector;
 using CameraInterop;
 using FaceAnalyser_Interop;
 using GazeAnalyser_Interop;
+using FaceDetectorInterop;
 using MediaReader;
-using System.Globalization;
 using Microsoft.WindowsAPICodePack.Dialogs;
 
 namespace OpenFaceOffline
@@ -86,7 +85,7 @@ namespace OpenFaceOffline
 
         // Some members for displaying the results
         private Capture capture;
-        private ImageReader image_reader;
+        private FaceDetector face_detector;
         private WriteableBitmap latest_img;
         private WriteableBitmap latest_aligned_face;
         private WriteableBitmap latest_HOG_descriptor;
@@ -154,6 +153,7 @@ namespace OpenFaceOffline
             face_analyser = new FaceAnalyserManaged(root, DynamicAUModels, image_output_size);
 
             gaze_analyser = new GazeAnalyserManaged();
+
         }
 
         // ----------------------------------------------------------
@@ -245,7 +245,132 @@ namespace OpenFaceOffline
 
         }
 
-        // 
+        // TODO here
+        private void ProcessIndividualImages(ImageReader reader) // TODO need interface for recording settings
+        {
+            // Make sure the GUI is setup appropriately
+            SetupFeatureExtractionMode();
+
+            // Indicate we will start running the thread
+            thread_running = true;
+
+            // Turn off unneeded visualisations and recording settings (this will change)
+            // TODO controlled by recorder settings, also all features will be done
+            bool TrackVid = ShowTrackedVideo; ShowTrackedVideo = true;
+            bool ShowApp = ShowAppearance; ShowAppearance = false;
+            bool ShowGeo = ShowGeometry; ShowGeometry = false;
+            bool showAU = ShowAUs; ShowAUs = false;
+            bool recAlign = RecordAligned;
+            bool recHOG = RecordHOG;
+
+            // Actually update the GUI accordingly
+            Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 2000), (Action)(() =>
+            {
+                VisualisationChange(null, null);
+            }));
+
+            // Setup the parameters optimized for working on individual images rather than sequences
+            face_model_params.optimiseForImages();
+
+            // Initialize the face detector if it has not been initialized yet
+            if(face_detector == null)
+            {
+                face_detector = new FaceDetector();
+            }
+
+            // Loading an image file (or a number of them)
+            while (reader.isOpened())
+            {
+                if (!thread_running)
+                {
+                    continue;
+                }
+
+                // Start the actual processing, TODO this should change?
+                Thread.CurrentThread.IsBackground = true;
+
+                clnf_model.Reset();
+                face_analyser.Reset();
+
+                var frame = new RawImage(reader.GetNextImage());
+                var grayFrame = new RawImage(reader.GetCurrentFrameGray());
+
+                double progress = reader.GetProgress();
+
+                // Detect faces here and return bounding boxes
+                List<Rect> face_detections = new List<Rect>();
+                List<double> confidences = new List<double>();
+                face_detector.DetectFacesHOG(face_detections, grayFrame, confidences);
+
+                //Rectangle
+                var landmark_detections = clnf_model.DetectMultiFaceLandmarksInImage(grayFrame, face_model_params);
+
+                // Go over all detected faces
+                for (int i = 0; i < landmark_detections.Count; ++i)
+                {
+                    // Predict action units
+                    var au_preds = face_analyser.PredictStaticAUs(grayFrame, landmark_detections[i]);
+                }
+
+                List<Point> landmark_points = new List<Point>();
+
+                for (int i = 0; i < landmark_detections.Count; ++i)
+                {
+
+                    List<Tuple<double, double>> landmarks = landmark_detections[i];
+                    foreach (var p in landmarks)
+                    {
+                        landmark_points.Add(new Point(p.Item1, p.Item2));
+                    }
+                }
+
+                // Visualisation TODO this should be lifted out? and actually be grabbed from the visualizer? rather than drawing points ourselves?
+                if (ShowTrackedVideo)
+                {
+                    Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
+                    {
+                        if (latest_img == null)
+                        {
+                            latest_img = frame.CreateWriteableBitmap();
+                        }
+
+                        frame.UpdateWriteableBitmap(latest_img);
+
+                        video.Source = latest_img;
+                        video.Confidence = 1;
+                        video.FPS = processing_fps.GetFPS();
+                        video.Progress = progress;
+
+                        video.OverlayLines = new List<Tuple<Point, Point>>();
+
+                        video.OverlayPoints = landmark_points;
+
+                    }));
+                }
+                latest_img = null;
+
+                // TODO how to report errors from the reader here? exceptions? logging? Problem for future versions?
+            }
+
+            // Clear image setup, restore the views, TODO this will change
+            ShowTrackedVideo = TrackVid;
+            ShowAppearance = ShowApp;
+            ShowGeometry = ShowGeo;
+            ShowAUs = showAU;
+            RecordHOG = recHOG;
+            RecordAligned = recAlign;
+
+            // Actually update the GUI accordingly
+            Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 2000), (Action)(() =>
+            {
+                VisualisationChange(null, null);
+            }));
+
+            EndMode();
+
+        }
+
+        // TODO old, remove
         private void ProcessImages(string[] filenames)
         {
             // Turn off unneeded visualisations and recording settings
@@ -836,12 +961,62 @@ namespace OpenFaceOffline
             }));
         }
 
-
+        // Some utilities for opening images/videos and directories
-        private void imageFileOpenClick(object sender, RoutedEventArgs e)
+        private ImageReader openMediaDialog(bool images)
         {
+            string[] image_files = new string[0];
+            Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 2, 0), (Action)(() =>
+            {
+                var d = new OpenFileDialog();
+                d.Multiselect = true;
+                if(images)
+                { 
+                    d.Filter = "Image files|*.jpg;*.jpeg;*.bmp;*.png;*.gif";
+                }
+                else
+                {
+                    d.Filter = "Video files|*.avi;*.wmv;*.mov;*.mpg;*.mpeg;*.mp4";
+                }
+                if (d.ShowDialog(this) == true)
+                {
+
+                    image_files = d.FileNames;
+
+                }
+            }));
+            List<string> img_files_list = new List<string>(image_files);
+            return new ImageReader(img_files_list);
+        }
+
+        private void individualImageFilesOpenClick(object sender, RoutedEventArgs e)
+        {
+            // First clean up existing tracking
+            StopTracking();
+
+            ImageReader reader = openMediaDialog(true);
+
+            processing_thread = new Thread(() => ProcessIndividualImages(reader));
+            processing_thread.Start();
+
+        }
+
+        private void individualImageDirectoryOpenClick(object sender, RoutedEventArgs e)
+        {
+
+            StopTracking();
+
+            // TODO open directory here
             new Thread(() => imageOpen()).Start();
         }
 
+        // TODO old
+        private void imageFileOpenClick(object sender, RoutedEventArgs e)
+        {
+            
+            new Thread(() => imageOpen()).Start();
+        }
+
+        // TODO old
         private void imageOpen()
         {
             StopTracking();

lib/local/CppInerop/CppInerop.vcxproj

@@ -183,6 +183,7 @@
     <ClInclude Include="GazeAnalyserInterop.h" />
     <ClInclude Include="ImageReader.h" />
     <ClInclude Include="LandmarkDetectorInterop.h" />
+    <ClInclude Include="FaceDetectorInterop.h" />
     <ClInclude Include="OpenCVWrappers.h" />
   </ItemGroup>
   <ItemGroup>

lib/local/CppInerop/CppInerop.vcxproj.filters

@@ -41,5 +41,8 @@
     <ClInclude Include="ImageReader.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="FaceDetectorInterop.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

lib/local/CppInerop/CppInterop.cpp

@@ -39,3 +39,4 @@
 #include "OpenCVWrappers.h"
 #include "CameraInterop.h"
 #include "ImageReader.h"
+#include "FaceDetectorInterop.h"

lib/local/CppInerop/ImageReader.h

@@ -85,12 +85,13 @@ namespace MediaReader {
 		ImageReader(System::String^ image_directory)
 		{
 			m_image_capture = new Utilities::ImageCapture();
+			m_is_opened = new bool;
 
 			std::string image_dir_std = msclr::interop::marshal_as<std::string>(image_directory);
 
 			*m_is_opened = m_image_capture->OpenDirectory(image_dir_std);
 
-			if (*m_is_opened)
+			if (!*m_is_opened)
 			{
 				throw gcnew ReadingFailedException("Failed to open a directory or an image");
 			}
@@ -99,6 +100,7 @@ namespace MediaReader {
 		ImageReader(System::Collections::Generic::List<System::String^>^ image_files)
 		{
 			m_image_capture = new Utilities::ImageCapture();
+			m_is_opened = new bool;
 
 			std::vector<std::string> image_files_std;
 
@@ -106,11 +108,12 @@ namespace MediaReader {
 			{
 				std::string image_file = msclr::interop::marshal_as<std::string>(image_files[i]);
 				image_files_std.push_back(image_file);
+
 			}
 
-			bool success = m_image_capture->OpenImageFiles(image_files_std);
+			*m_is_opened = m_image_capture->OpenImageFiles(image_files_std);
 
-			if (!success)
+			if (!*m_is_opened)
 			{
 				throw gcnew ReadingFailedException("Failed to open a directory or an image");
 			}
@@ -128,6 +131,11 @@ namespace MediaReader {
 
 			next_image.copyTo(m_rgb_frame->Mat);
 
+			if (next_image.empty())
+			{
+				*m_is_opened = false;
+			}
+
 			return m_rgb_frame;
 		}