diff --git a/gui/OpenFaceOffline/MainWindow.xaml.cs b/gui/OpenFaceOffline/MainWindow.xaml.cs index 3ffa39b..a6a787d 100644 --- a/gui/OpenFaceOffline/MainWindow.xaml.cs +++ b/gui/OpenFaceOffline/MainWindow.xaml.cs @@ -83,7 +83,6 @@ namespace OpenFaceOffline Thread processing_thread; // Some members for displaying the results - private Capture capture; private WriteableBitmap latest_img; private WriteableBitmap latest_aligned_face; private WriteableBitmap latest_HOG_descriptor; @@ -108,9 +107,6 @@ namespace OpenFaceOffline FaceAnalyserManaged face_analyser; GazeAnalyserManaged gaze_analyser; - // For output recording - Recorder recorder; - public bool RecordAligned { get; set; } = false; // Aligned face images public bool RecordHOG { get; set; } = false; // HOG features extracted from face images public bool Record2DLandmarks { get; set; } = true; // 2D locations of facial landmarks (in pixels) @@ -160,82 +156,90 @@ namespace OpenFaceOffline // ---------------------------------------------------------- // Actual work gets done here - // The main function call for processing images or video files, TODO rename this as it is not a loop - private void ProcessingLoop(String[] filenames, int cam_id = -1, int width = -1, int height = -1, bool multi_face = false) + // Wrapper for processing multiple sequences + private void ProcessSequences(List filenames) + { + for (int i = 0; i < filenames.Count; ++i) + { + SequenceReader reader = new SequenceReader(filenames[i], false); + ProcessSequence(reader); + } + + } + + // The main function call for processing sequences + private void ProcessSequence(SequenceReader reader) { SetupFeatureExtractionMode(); thread_running = true; - // Create the video capture and call the VideoLoop - if (filenames != null) + face_model_params.optimiseForVideo(); + + // Setup the visualization + Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance); + + // Initialize the face analyser + face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, DynamicAUModels, image_output_size); + + // Reset the tracker + clnf_model.Reset(); + + // Loading an image file + var frame = new RawImage(reader.GetNextImage()); + var gray_frame = new RawImage(reader.GetCurrentFrameGray()); + + // Setup recording + RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(true, reader.IsWebcam(), + Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs, + RecordGaze, RecordHOG, RecordTracked, RecordAligned, + reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetFPS()); + + RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root); + + // For FPS tracking + DateTime? startTime = CurrentTime; + var lastFrameTime = CurrentTime; + + // This will be false when the image is not available + while (reader.IsOpened()) { - face_model_params.optimiseForVideo(); - if (cam_id == -2) + if(!thread_running) { - List image_files_all = new List(); - foreach (string image_name in filenames) - image_files_all.Add(image_name); - - // Loading an image sequence that represents a video - capture = new Capture(image_files_all); - - if (capture.isOpened()) - { - // Prepare recording if any based on the directory - String file_no_ext = System.IO.Path.GetDirectoryName(filenames[0]); - file_no_ext = System.IO.Path.GetFileName(file_no_ext); - - // Start the actual processing and recording - FeatureExtractionLoop(file_no_ext); - - } - else - { - string messageBoxText = "Failed to open an image"; - string caption = "Not valid file"; - MessageBoxButton button = MessageBoxButton.OK; - MessageBoxImage icon = MessageBoxImage.Warning; - - // Display message box - System.Windows.MessageBox.Show(messageBoxText, caption, button, icon); - } + continue; } - else + + lastFrameTime = CurrentTime; + processing_fps.AddFrame(); + double progress = reader.GetProgress(); + + bool detectionSucceeding = clnf_model.DetectLandmarksInVideo(gray_frame, face_model_params); + + // The face analysis step (for AUs and eye gaze) + face_analyser.AddNextFrame(frame, clnf_model.CalculateAllLandmarks(), detectionSucceeding, false); + gaze_analyser.AddNextFrame(clnf_model, detectionSucceeding, fx, fy, cx, cy); + + // Only the final face will contain the details + VisualizeFeatures(frame, visualizer_of, clnf_model.CalculateAllLandmarks(), true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); + + // Record an observation + RecordObservation(recorder, visualizer_of.GetVisImage(), detectionSucceeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); + + while (thread_running & thread_paused && skip_frames == 0) { - face_model_params.optimiseForVideo(); - // Loading a video file (or a number of them) - foreach (string filename in filenames) - { - if (!thread_running) - { - continue; - } - - capture = new Capture(filename); - - if (capture.isOpened()) - { - String file_no_ext = System.IO.Path.GetFileNameWithoutExtension(filename); - - // Start the actual processing - FeatureExtractionLoop(file_no_ext); - - } - else - { - string messageBoxText = "File is not a video or the codec is not supported."; - string caption = "Not valid file"; - MessageBoxButton button = MessageBoxButton.OK; - MessageBoxImage icon = MessageBoxImage.Warning; - - // Display message box - System.Windows.MessageBox.Show(messageBoxText, caption, button, icon); - } - } + Thread.Sleep(10); } + + if (skip_frames > 0) + skip_frames--; + + frame = new RawImage(reader.GetNextImage()); + gray_frame = new RawImage(reader.GetCurrentFrameGray()); } + // Post-process the AU recordings, TODO + //recorder.FinishRecording(clnf_model, face_analyser); + EndMode(); } @@ -329,110 +333,6 @@ namespace OpenFaceOffline } - // Capturing and processing the video frame by frame - private void FeatureExtractionLoop(string output_file_name) - { - - DateTime? startTime = CurrentTime; - - var lastFrameTime = CurrentTime; - - clnf_model.Reset(); - face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, DynamicAUModels, image_output_size); - - // If the camera calibration parameters are not set (indicated by -1), guesstimate them - if(estimate_camera_parameters || fx == -1 || fy == -1 || cx == -1 || cy == -1) - { - fx = 500.0f * (capture.width / 640.0f); - fy = 500.0f * (capture.height / 480.0f); - - fx = (fx + fy) / 2.0f; - fy = fx; - - cx = capture.width / 2f; - cy = capture.height / 2f; - } - - // Setup the recorder first - recorder = new Recorder(record_root, output_file_name, capture.width, capture.height, Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, - RecordAUs, RecordGaze, RecordAligned, RecordHOG, clnf_model, face_analyser, fx, fy, cx, cy, DynamicAUModels); - - // Setup the c++ visualizer - Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance); - - int frame_id = 0; - - double fps = capture.GetFPS(); - if (fps <= 0) fps = 30; - - while (thread_running) - { - ////////////////////////////////////////////// - // CAPTURE FRAME AND DETECT LANDMARKS FOLLOWED BY THE REQUIRED IMAGE PROCESSING - ////////////////////////////////////////////// - RawImage frame = null; - double progress = -1; - - frame = new RawImage(capture.GetNextFrame(false)); - progress = capture.GetProgress(); - - if (frame.Width == 0) - { - // This indicates that we reached the end of the video file - break; - } - - lastFrameTime = CurrentTime; - processing_fps.AddFrame(); - - var grayFrame = new RawImage(capture.GetCurrentFrameGray()); - - if (grayFrame == null) - { - Console.WriteLine("Gray is empty"); - continue; - } - - detectionSucceeding = ProcessFrame(clnf_model, face_model_params, frame, grayFrame, fx, fy, cx, cy); - - // The face analysis step (for AUs and eye gaze) - face_analyser.AddNextFrame(frame, clnf_model.CalculateAllLandmarks(), detectionSucceeding, false); - gaze_analyser.AddNextFrame(clnf_model, detectionSucceeding, fx, fy, cx, cy); - - recorder.RecordFrame(clnf_model, face_analyser, gaze_analyser, detectionSucceeding, frame_id + 1, ((double)frame_id) / fps); - - List> landmarks = clnf_model.CalculateVisibleLandmarks(); - - VisualizeFeatures(frame, visualizer_of, landmarks, true, fx, fy, cx, cy, progress); - - while (thread_running & thread_paused && skip_frames == 0) - { - Thread.Sleep(10); - } - - frame_id++; - - if (skip_frames > 0) - skip_frames--; - - } - - latest_img = null; - skip_frames = 0; - - // Unpause if it's paused - if (thread_paused) - { - Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() => - { - PauseButton_Click(null, null); - })); - } - - recorder.FinishRecording(clnf_model, face_analyser); - - } - private void RecordObservation(RecorderOpenFace recorder, RawImage vis_image, bool success, float fx, float fy, float cx, float cy) { @@ -651,17 +551,6 @@ namespace OpenFaceOffline } - - // ---------------------------------------------------------- - // Interacting with landmark detection and face analysis - - private bool ProcessFrame(CLNF clnf_model, FaceModelParameters clnf_params, RawImage frame, RawImage grayscale_frame, double fx, double fy, double cx, double cy) - { - detectionSucceeding = clnf_model.DetectLandmarksInVideo(grayscale_frame, clnf_params); - return detectionSucceeding; - - } - // ---------------------------------------------------------- // Mode handling (image, video) // ---------------------------------------------------------- @@ -685,6 +574,18 @@ namespace OpenFaceOffline // When the processing is done re-enable the components private void EndMode() { + latest_img = null; + skip_frames = 0; + + // Unpause if it's paused + if (thread_paused) + { + Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() => + { + PauseButton_Click(null, null); + })); + } + Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 1, 0), (Action)(() => { @@ -725,43 +626,16 @@ namespace OpenFaceOffline // Opening Videos/Images // ---------------------------------------------------------- - private void videoFileOpenClick(object sender, RoutedEventArgs e) - { - new Thread(() => openVideoFile()).Start(); - } - - private void openVideoFile() - { - StopTracking(); - - Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 2, 0), (Action)(() => - { - var d = new Microsoft.Win32.OpenFileDialog(); - d.Multiselect = true; - d.Filter = "Video files|*.avi;*.wmv;*.mov;*.mpg;*.mpeg;*.mp4"; - - if (d.ShowDialog(this) == true) - { - - string[] video_files = d.FileNames; - - processing_thread = new Thread(() => ProcessingLoop(video_files)); - processing_thread.Start(); - - } - })); - } - // Some utilities for opening images/videos and directories - private ImageReader openMediaDialog(bool images) + private List openMediaDialog(bool images) { string[] image_files = new string[0]; Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 2, 0), (Action)(() => { var d = new Microsoft.Win32.OpenFileDialog(); d.Multiselect = true; - if(images) - { + if (images) + { d.Filter = "Image files|*.jpg;*.jpeg;*.bmp;*.png;*.gif"; } else @@ -776,9 +650,9 @@ namespace OpenFaceOffline } })); List img_files_list = new List(image_files); - return new ImageReader(img_files_list); + return img_files_list; } - + private string openDirectory() { string to_return = ""; @@ -805,13 +679,41 @@ namespace OpenFaceOffline return to_return; } + private void imageSequenceFileOpenClick(object sender, RoutedEventArgs e) + { + // First clean up existing tracking + StopTracking(); + + string directory = openDirectory(); + if (!string.IsNullOrWhiteSpace(directory)) + { + SequenceReader reader = new SequenceReader(directory, true); + + processing_thread = new Thread(() => ProcessSequence(reader)); + processing_thread.Start(); + } + + } + + private void videoFileOpenClick(object sender, RoutedEventArgs e) + { + // First clean up existing tracking + StopTracking(); + + var video_files = openMediaDialog(false); + processing_thread = new Thread(() => ProcessSequences(video_files)); + processing_thread.Start(); + + } + // Selecting one or more images in a directory private void individualImageFilesOpenClick(object sender, RoutedEventArgs e) { // First clean up existing tracking StopTracking(); - ImageReader reader = openMediaDialog(true); + var image_files = openMediaDialog(true); + ImageReader reader = new ImageReader(image_files); processing_thread = new Thread(() => ProcessIndividualImages(reader)); processing_thread.Start(); @@ -835,35 +737,6 @@ namespace OpenFaceOffline } } - - private void imageSequenceFileOpenClick(object sender, RoutedEventArgs e) - { - new Thread(() => imageSequenceOpen()).Start(); - } - - // TODO this should be removed and replace with directory open - private void imageSequenceOpen() - { - StopTracking(); - - Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 2, 0), (Action)(() => - { - var d = new Microsoft.Win32.OpenFileDialog(); - d.Multiselect = true; - d.Filter = "Image files|*.jpg;*.jpeg;*.bmp;*.png;*.gif"; - - if (d.ShowDialog(this) == true) - { - - string[] image_files = d.FileNames; - - processing_thread = new Thread(() => ProcessingLoop(image_files, -2)); - processing_thread.Start(); - - } - })); - } - // -------------------------------------------------------- // Button handling // -------------------------------------------------------- @@ -876,8 +749,6 @@ namespace OpenFaceOffline // Stop capture and tracking thread_running = false; processing_thread.Join(); - - capture.Dispose(); } face_analyser.Dispose(); } diff --git a/gui/OpenFaceOffline/OpenFaceOffline.csproj b/gui/OpenFaceOffline/OpenFaceOffline.csproj index 1b20fa3..b1e7bdc 100644 --- a/gui/OpenFaceOffline/OpenFaceOffline.csproj +++ b/gui/OpenFaceOffline/OpenFaceOffline.csproj @@ -88,7 +88,6 @@ Designer - BarGraph.xaml diff --git a/gui/OpenFaceOffline/Recorder.cs b/gui/OpenFaceOffline/Recorder.cs deleted file mode 100644 index bbc649d..0000000 --- a/gui/OpenFaceOffline/Recorder.cs +++ /dev/null @@ -1,305 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, -// all rights reserved. -// -// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY -// -// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. -// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. -// -// License can be found in OpenFace-license.txt - -// * Any publications arising from the use of this software, including but -// not limited to academic journal and conference publications, technical -// reports and manuals, must cite at least one of the following works: -// -// OpenFace: an open source facial behavior analysis toolkit -// Tadas BaltruÅ¡aitis, Peter Robinson, and Louis-Philippe Morency -// in IEEE Winter Conference on Applications of Computer Vision, 2016 -// -// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation -// Erroll Wood, Tadas BaltruÅ¡aitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling -// in IEEE International. Conference on Computer Vision (ICCV), 2015 -// -// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection -// Tadas BaltruÅ¡aitis, Marwa Mahmoud, and Peter Robinson -// in Facial Expression Recognition and Analysis Challenge, -// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 -// -// Constrained Local Neural Fields for robust facial landmark detection in the wild. -// Tadas BaltruÅ¡aitis, Peter Robinson, and Louis-Philippe Morency. -// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. -// -/////////////////////////////////////////////////////////////////////////////// - -using CppInterop.LandmarkDetector; -using FaceAnalyser_Interop; -using GazeAnalyser_Interop; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace OpenFaceOffline -{ - class Recorder - { - StreamWriter output_features_file; - - bool output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, record_aligned, record_HOG; - - double fx, fy, cx, cy; - - List au_reg_names; - List au_class_names; - - String out_filename; - - bool dynamic_AU_model; - - public Recorder(string root, string filename, int width, int height, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, - bool output_pose, bool output_AUs, bool output_gaze, bool record_aligned, bool record_HOG, - CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy, bool dynamic_AU_model) - { - - this.output_2D_landmarks = output_2D_landmarks; this.output_3D_landmarks = output_3D_landmarks; - this.output_model_params = output_model_params; this.output_pose = output_pose; - this.output_AUs = output_AUs; this.output_gaze = output_gaze; - this.record_aligned = record_aligned; this.record_HOG = record_HOG; - - this.fx = fx; this.fy = fy; this.cx = cx; this.cy = cy; - - this.dynamic_AU_model = dynamic_AU_model; - - if (!System.IO.Directory.Exists(root)) - { - System.IO.Directory.CreateDirectory(root); - } - - // Write out the OF file which tells where all the relevant data is - StreamWriter out_of_file = new StreamWriter(root + "/" + filename + ".of"); - - //out_of_file.WriteLine("Video_file:" + ) - out_of_file.WriteLine("CSV file: " + root + "/" + filename + ".csv"); - if(record_HOG) - { - out_of_file.WriteLine("HOG file: " + root + "/" + filename + ".hog"); - } - if(record_aligned) - { - out_of_file.WriteLine("Aligned dir: " + root + "/" + filename + "/"); - } - - out_filename = root + "/" + filename + ".csv"; - output_features_file = new StreamWriter(out_filename); - output_features_file.Write("frame, timestamp, confidence, success"); - - if (output_gaze) - { - output_features_file.Write(", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y"); - - // Output gaze eye landmarks - int gaze_num_lmks = clnf_model.CalculateAllEyeLandmarks().Count; - for (int i = 0; i < gaze_num_lmks; ++i) - { - output_features_file.Write(", eye_lmk_x_" + i); - } - for (int i = 0; i < gaze_num_lmks; ++i) - { - output_features_file.Write(", eye_lmk_y_" + i); - } - } - - if (output_pose) - output_features_file.Write(", pose_Tx, pose_Ty, pose_Tz, pose_Rx, pose_Ry, pose_Rz"); - - if (output_2D_landmarks) - { - for (int i = 0; i < clnf_model.GetNumPoints(); ++i) - { - output_features_file.Write(", x_" + i); - } - for (int i = 0; i < clnf_model.GetNumPoints(); ++i) - { - output_features_file.Write(", y_" + i); - } - } - - if (output_3D_landmarks) - { - - for (int i = 0; i < clnf_model.GetNumPoints(); ++i) - { - output_features_file.Write(", X_" + i); - } - for (int i = 0; i < clnf_model.GetNumPoints(); ++i) - { - output_features_file.Write(", Y_" + i); - } - for (int i = 0; i < clnf_model.GetNumPoints(); ++i) - { - output_features_file.Write(", Z_" + i); - } - } - - if (output_model_params) - { - output_features_file.Write(", p_scale, p_rx, p_ry, p_rz, p_tx, p_ty"); - for (int i = 0; i < clnf_model.GetNumModes(); ++i) - { - output_features_file.Write(", p_" + i); - } - } - - if (output_AUs) - { - - au_reg_names = face_analyser.GetRegActionUnitsNames(); - au_reg_names.Sort(); - foreach (var name in au_reg_names) - { - output_features_file.Write(", " + name + "_r"); - } - - au_class_names = face_analyser.GetClassActionUnitsNames(); - au_class_names.Sort(); - foreach (var name in au_class_names) - { - output_features_file.Write(", " + name + "_c"); - } - - } - - output_features_file.WriteLine(); - - if (record_aligned) - { - String aligned_root = root + "/" + filename + "_aligned/"; - System.IO.Directory.CreateDirectory(aligned_root); - face_analyser.SetupAlignedImageRecording(aligned_root); - } - - if (record_HOG) - { - String filename_HOG = root + "/" + filename + ".hog"; - face_analyser.SetupHOGRecording(filename_HOG); - } - } - - public void RecordFrame(CLNF clnf_model, FaceAnalyserManaged face_analyser, GazeAnalyserManaged gaze_analyser, bool success, int frame_ind, double time_stamp) - { - // Making sure that full stop is used instead of a comma for data recording - System.Globalization.CultureInfo customCulture = (System.Globalization.CultureInfo)System.Threading.Thread.CurrentThread.CurrentCulture.Clone(); - customCulture.NumberFormat.NumberDecimalSeparator = "."; - - System.Threading.Thread.CurrentThread.CurrentCulture = customCulture; - - double confidence = (-clnf_model.GetConfidence()) / 2.0 + 0.5; - - List pose = new List(); - clnf_model.GetPose(pose, fx, fy, cx, cy); - - output_features_file.Write(String.Format("{0}, {1}, {2:F3}, {3}", frame_ind, time_stamp, confidence, success ? 1 : 0)); - - if (output_gaze) - { - var gaze = gaze_analyser.GetGazeCamera(); - var gaze_angle = gaze_analyser.GetGazeAngle(); - - output_features_file.Write(String.Format(", {0:F5}, {1:F5}, {2:F5}, {3:F5}, {4:F5}, {5:F5}, {6:F5}, {7:F5}", gaze.Item1.Item1, gaze.Item1.Item2, gaze.Item1.Item3, - gaze.Item2.Item1, gaze.Item2.Item2, gaze.Item2.Item3, gaze_angle.Item1, gaze_angle.Item2)); - - List> landmarks_2d = clnf_model.CalculateAllEyeLandmarks(); - - for (int i = 0; i < landmarks_2d.Count; ++i) - output_features_file.Write(", {0:F3}", landmarks_2d[i].Item1); - - for (int i = 0; i < landmarks_2d.Count; ++i) - output_features_file.Write(", {0:F3}", landmarks_2d[i].Item2); - - } - - if (output_pose) - output_features_file.Write(String.Format(", {0:F4}, {1:F4}, {2:F4}, {3:F4}, {4:F4}, {5:F4}", pose[0], pose[1], pose[2], pose[3], pose[4], pose[5])); - - if (output_2D_landmarks) - { - List> landmarks_2d = clnf_model.CalculateAllLandmarks(); - - for (int i = 0; i < landmarks_2d.Count; ++i) - output_features_file.Write(", {0:F3}", landmarks_2d[i].Item1); - - for (int i = 0; i < landmarks_2d.Count; ++i) - output_features_file.Write(", {0:F3}", landmarks_2d[i].Item2); - } - - if (output_3D_landmarks) - { - List> landmarks_3d = clnf_model.Calculate3DLandmarks(fx, fy, cx, cy); - - for (int i = 0; i < landmarks_3d.Count; ++i) - output_features_file.Write(", {0:F3}", landmarks_3d[i].Item1); - - for (int i = 0; i < landmarks_3d.Count; ++i) - output_features_file.Write(", {0:F3}", landmarks_3d[i].Item2); - - for (int i = 0; i < landmarks_3d.Count; ++i) - output_features_file.Write(", {0:F3}", landmarks_3d[i].Item3); - } - - if (output_model_params) - { - List all_params = clnf_model.GetParams(); - - for (int i = 0; i < all_params.Count; ++i) - output_features_file.Write(String.Format(", {0,0:F5}", all_params[i])); - } - - if (output_AUs) - { - var au_regs = face_analyser.GetCurrentAUsReg(); - - foreach (var name_reg in au_reg_names) - output_features_file.Write(", {0:F2}", au_regs[name_reg]); - - var au_classes = face_analyser.GetCurrentAUsClass(); - - foreach (var name_class in au_class_names) - output_features_file.Write(", {0:F0}", au_classes[name_class]); - - } - - output_features_file.WriteLine(); - - if (record_aligned) - { - face_analyser.RecordAlignedFrame(frame_ind); - } - - if (record_HOG) - { - face_analyser.RecordHOGFrame(); - } - - } - - public void FinishRecording(CLNF clnf_model, FaceAnalyserManaged face_analyser) - { - if (output_features_file != null) - output_features_file.Close(); - - if (record_HOG) - face_analyser.StopHOGRecording(); - - face_analyser.PostProcessOutputFile(out_filename); - } - - static void RecordImg(string out_root, string filename, CLNF clnf_model, FaceAnalyserManaged face_analyser, double fx, double fy, double cx, double cy) - { - // Points, pose, gaze, aus - } - - } -} diff --git a/lib/local/CppInerop/CppInerop.vcxproj b/lib/local/CppInerop/CppInerop.vcxproj index c1d5fcc..f07cb0a 100644 --- a/lib/local/CppInerop/CppInerop.vcxproj +++ b/lib/local/CppInerop/CppInerop.vcxproj @@ -186,6 +186,7 @@ + diff --git a/lib/local/CppInerop/CppInerop.vcxproj.filters b/lib/local/CppInerop/CppInerop.vcxproj.filters index dd35bd0..5ce516c 100644 --- a/lib/local/CppInerop/CppInerop.vcxproj.filters +++ b/lib/local/CppInerop/CppInerop.vcxproj.filters @@ -50,5 +50,8 @@ Header Files + + Header Files + \ No newline at end of file diff --git a/lib/local/CppInerop/CppInterop.cpp b/lib/local/CppInerop/CppInterop.cpp index f60fcc0..e0271f4 100644 --- a/lib/local/CppInerop/CppInterop.cpp +++ b/lib/local/CppInerop/CppInterop.cpp @@ -41,4 +41,5 @@ #include "ImageReader.h" #include "FaceDetectorInterop.h" #include "RecorderInterop.h" -#include "VisualizerInterop.h" \ No newline at end of file +#include "VisualizerInterop.h" +#include "SequenceReader.h" \ No newline at end of file diff --git a/lib/local/CppInerop/ImageReader.h b/lib/local/CppInerop/ImageReader.h index fee6e07..d0e74c8 100644 --- a/lib/local/CppInerop/ImageReader.h +++ b/lib/local/CppInerop/ImageReader.h @@ -1,6 +1,5 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, -// all rights reserved. +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. // // ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY // @@ -32,8 +31,6 @@ // /////////////////////////////////////////////////////////////////////////////// -// Camera_Interop.h - #pragma once #pragma unmanaged @@ -210,7 +207,7 @@ namespace UtilitiesOF { } if (m_is_opened != nullptr) { - delete m_gray_frame; + delete m_is_opened; } } diff --git a/lib/local/CppInerop/SequenceReader.h b/lib/local/CppInerop/SequenceReader.h new file mode 100644 index 0000000..7074741 --- /dev/null +++ b/lib/local/CppInerop/SequenceReader.h @@ -0,0 +1,214 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt + +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#pragma unmanaged + +// Include all the unmanaged things we need. + +#include +#include "opencv2/objdetect.hpp" +#include "opencv2/calib3d.hpp" +#include +#include +#include +#include +#include + +#include +#include + +#include "SequenceCapture.h" + +#pragma managed + +#include +#include + +namespace UtilitiesOF { + + + public ref class SequenceReader + { + private: + + // OpenCV based video capture for reading from files + Utilities::SequenceCapture* m_sequence_capture; + + OpenCVWrappers::RawImage^ m_rgb_frame; + OpenCVWrappers::RawImage^ m_gray_frame; + + public: + + // Can provide a directory or a video filename, need to specify which + SequenceReader(System::String^ filename, bool directory) + { + m_sequence_capture = new Utilities::SequenceCapture(); + + std::string name_std = msclr::interop::marshal_as(filename); + + bool success; + + if(directory) + { + success = m_sequence_capture->OpenImageSequence(name_std); + } + else + { + success = m_sequence_capture->OpenVideoFile(name_std); + } + + if (!success) + { + throw gcnew ReadingFailedException("Failed to open an image sequence"); + } + } + + // Can provide a webcam id + SequenceReader(int webcam_id) + { + m_sequence_capture = new Utilities::SequenceCapture(); + + bool success = m_sequence_capture->OpenWebcam(webcam_id); + + if (!success) + { + throw gcnew ReadingFailedException("Failed to open an image sequence"); + } + } + + OpenCVWrappers::RawImage^ GetNextImage() + { + cv::Mat next_image = m_sequence_capture->GetNextFrame(); + + if (m_rgb_frame == nullptr) + { + m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image.size().width, next_image.size().width, CV_8UC3); + } + + next_image.copyTo(m_rgb_frame->Mat); + + return m_rgb_frame; + } + + System::String^ GetName() + { + std::string filename = m_sequence_capture->name; + return gcnew System::String(filename.c_str()); + } + + double GetProgress() + { + return m_sequence_capture->GetProgress(); + } + + float GetFx() + { + return m_sequence_capture->fx; + } + + float GetFy() + { + return m_sequence_capture->fy; + } + + float GetCx() + { + return m_sequence_capture->cx; + } + + float GetCy() + { + return m_sequence_capture->cy; + } + + bool IsOpened() + { + return m_sequence_capture->IsOpened(); + } + + bool IsWebcam() + { + return m_sequence_capture->IsWebcam(); + } + + double GetFPS() + { + return m_sequence_capture->fps; + } + + OpenCVWrappers::RawImage^ GetCurrentFrameGray() { + + cv::Mat next_gray_image = m_sequence_capture->GetGrayFrame(); + + if (m_gray_frame == nullptr) + { + m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image.size().width, next_gray_image.size().width, CV_8UC3); + } + + next_gray_image.copyTo(m_gray_frame->Mat); + + return m_gray_frame; + } + + // Finalizer. Definitely called before Garbage Collection, + // but not automatically called on explicit Dispose(). + // May be called multiple times. + !SequenceReader() + { + // Automatically closes capture object before freeing memory. + if (m_sequence_capture != nullptr) + { + delete m_sequence_capture; + } + + if (m_rgb_frame != nullptr) + { + delete m_rgb_frame; + } + if (m_gray_frame != nullptr) + { + delete m_gray_frame; + } + + } + + // Destructor. Called on explicit Dispose() only. + ~SequenceReader() + { + this->!SequenceReader(); + } + }; + +}