sustaining_gazes/gui/OpenFaceOffline/MainWindow.xaml.cs

971 lines
38 KiB
C#
Raw Normal View History

2016-05-20 22:48:43 +02:00
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
2016-05-20 22:48:43 +02:00
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
2016-05-20 22:48:43 +02:00
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
2016-05-20 22:48:43 +02:00
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace: an open source facial behavior analysis toolkit
// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency
// in IEEE Winter Conference on Applications of Computer Vision, 2016
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
// Constrained Local Neural Fields for robust facial landmark detection in the wild.
// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency.
// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013.
//
///////////////////////////////////////////////////////////////////////////////
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading;
using System.Windows;
using System.Windows.Threading;
using System.Windows.Media.Imaging;
// Internal libraries
using OpenCVWrappers;
using CppInterop.LandmarkDetector;
using FaceAnalyser_Interop;
using GazeAnalyser_Interop;
2018-01-09 18:08:39 +01:00
using FaceDetectorInterop;
using UtilitiesOF;
using Microsoft.WindowsAPICodePack.Dialogs;
using System.Windows.Forms;
2016-05-20 22:48:43 +02:00
namespace OpenFaceOffline
{
/// <summary>
/// Interaction logic for MainWindow.xaml
/// </summary>
public partial class MainWindow : Window
{
// Timing for measuring FPS
#region High-Resolution Timing
static DateTime startTime;
static Stopwatch sw = new Stopwatch();
static MainWindow()
{
startTime = DateTime.Now;
sw.Start();
}
public static DateTime CurrentTime
{
get { return startTime + sw.Elapsed; }
}
#endregion
// -----------------------------------------------------------------
// Members
// -----------------------------------------------------------------
Thread processing_thread;
// Some members for displaying the results
private WriteableBitmap latest_img;
private WriteableBitmap latest_aligned_face;
private WriteableBitmap latest_HOG_descriptor;
// Managing the running of the analysis system
private volatile bool thread_running;
private volatile bool thread_paused = false;
// Allows for going forward in time step by step
// Useful for visualising things
private volatile int skip_frames = 0;
FpsTracker processing_fps = new FpsTracker();
// For selecting webcams
CameraSelection cam_sec;
2016-05-20 22:48:43 +02:00
// For tracking
FaceDetector face_detector;
FaceModelParameters face_model_params;
2018-01-25 09:13:02 +01:00
CLNF landmark_detector;
// For face analysis
2018-01-10 09:27:16 +01:00
FaceAnalyserManaged face_analyser;
GazeAnalyserManaged gaze_analyser;
2016-05-20 22:48:43 +02:00
public bool RecordAligned { get; set; } = false; // Aligned face images
public bool RecordHOG { get; set; } = false; // HOG features extracted from face images
public bool Record2DLandmarks { get; set; } = true; // 2D locations of facial landmarks (in pixels)
public bool Record3DLandmarks { get; set; } = true; // 3D locations of facial landmarks (in pixels)
public bool RecordModelParameters { get; set; } = true; // Facial shape parameters (rigid and non-rigid geometry)
public bool RecordPose { get; set; } = true; // Head pose (position and orientation)
public bool RecordAUs { get; set; } = true; // Facial action units
public bool RecordGaze { get; set; } = true; // Eye gaze
2018-01-27 08:46:21 +01:00
public bool RecordTracked { get; set; } = true; // Recording tracked videos or images
2016-05-20 23:38:03 +02:00
2016-05-20 22:48:43 +02:00
// Visualisation options
public bool ShowTrackedVideo { get; set; } = true; // Showing the actual tracking
public bool ShowAppearance { get; set; } = true; // Showing appeaance features like HOG
public bool ShowGeometry { get; set; } = true; // Showing geometry features, pose, gaze, and non-rigid
public bool ShowAUs { get; set; } = true; // Showing Facial Action Units
int image_output_size = 112;
public bool MaskAligned { get; set; } = true; // Should the aligned images be masked
// Where the recording is done (by default in a record directory, from where the application executed)
String record_root = "./processed";
// For AU prediction, if videos are long dynamic models should be used
public bool DynamicAUModels { get; set; } = true;
// Camera calibration parameters
2018-01-21 11:08:24 +01:00
public float fx = -1, fy = -1, cx = -1, cy = -1;
2016-05-20 22:48:43 +02:00
public MainWindow()
{
InitializeComponent();
this.DataContext = this; // For WPF data binding
2016-05-20 22:48:43 +02:00
// Set the icon
Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);
this.Icon = BitmapFrame.Create(iconUri);
2016-12-08 22:43:29 +01:00
2016-05-20 22:48:43 +02:00
String root = AppDomain.CurrentDomain.BaseDirectory;
face_model_params = new FaceModelParameters(root, false);
2018-01-25 09:13:02 +01:00
landmark_detector = new CLNF(face_model_params);
2016-05-20 22:48:43 +02:00
gaze_analyser = new GazeAnalyserManaged();
2018-01-09 18:08:39 +01:00
2016-05-20 22:48:43 +02:00
}
// ----------------------------------------------------------
// Actual work gets done here
// Wrapper for processing multiple sequences
private void ProcessSequences(List<String> filenames)
{
for (int i = 0; i < filenames.Count; ++i)
{
SequenceReader reader = new SequenceReader(filenames[i], false, fx, fy, cx, cy);
ProcessSequence(reader);
// Before continuing to next video make sure the user did not stop the processing
2018-01-27 08:46:21 +01:00
if (!thread_running)
{
break;
}
}
}
// The main function call for processing sequences
private void ProcessSequence(SequenceReader reader)
2016-05-20 22:48:43 +02:00
{
Thread.CurrentThread.Priority = ThreadPriority.Highest;
SetupFeatureExtractionMode();
2016-05-20 22:48:43 +02:00
thread_running = true;
face_model_params.optimiseForVideo();
2016-05-20 22:48:43 +02:00
// Setup the visualization
Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance);
2016-05-20 22:48:43 +02:00
// Initialize the face analyser
face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, DynamicAUModels, image_output_size, MaskAligned);
// Reset the tracker
2018-01-25 09:13:02 +01:00
landmark_detector.Reset();
2016-05-20 22:48:43 +02:00
// Loading an image file
var frame = new RawImage(reader.GetNextImage());
var gray_frame = new RawImage(reader.GetCurrentFrameGray());
2016-05-20 22:48:43 +02:00
// Setup recording
RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(true, reader.IsWebcam(),
Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs,
RecordGaze, RecordHOG, RecordTracked, RecordAligned,
reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetFPS());
RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root);
// For FPS tracking
DateTime? startTime = CurrentTime;
var lastFrameTime = CurrentTime;
// Empty image would indicate that the stream is over
while (gray_frame.Width != 0)
{
if(!thread_running)
2016-05-20 22:48:43 +02:00
{
break;
}
2016-05-20 22:48:43 +02:00
double progress = reader.GetProgress();
2018-01-25 09:13:02 +01:00
bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(gray_frame, face_model_params);
2016-05-20 22:48:43 +02:00
// The face analysis step (for AUs and eye gaze)
2018-01-25 09:13:02 +01:00
face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, false);
gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
2016-05-20 22:48:43 +02:00
// Only the final face will contain the details
2018-01-31 09:23:48 +01:00
VisualizeFeatures(frame, visualizer_of, landmark_detector.CalculateAllLandmarks(), landmark_detector.GetVisibilities(), detection_succeeding, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress);
// Record an observation
2018-01-28 11:16:20 +01:00
RecordObservation(recorder, visualizer_of.GetVisImage(), detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetTimestamp());
while (thread_running & thread_paused && skip_frames == 0)
{
Thread.Sleep(10);
2016-05-20 22:48:43 +02:00
}
if (skip_frames > 0)
skip_frames--;
frame = new RawImage(reader.GetNextImage());
gray_frame = new RawImage(reader.GetCurrentFrameGray());
lastFrameTime = CurrentTime;
processing_fps.AddFrame();
2016-05-20 22:48:43 +02:00
}
2018-01-27 08:46:21 +01:00
// Finalize the recording and flush to disk
recorder.Close();
// Post-process the AU recordings
2018-01-29 18:26:35 +01:00
if(RecordAUs)
{
face_analyser.PostProcessOutputFile(recorder.GetCSVFile());
}
2018-01-27 08:46:21 +01:00
// Close the open video/webcam
reader.Close();
EndMode();
2016-05-20 22:48:43 +02:00
}
private void ProcessIndividualImages(ImageReader reader)
2018-01-09 18:08:39 +01:00
{
// Make sure the GUI is setup appropriately
SetupFeatureExtractionMode();
// Indicate we will start running the thread
thread_running = true;
// Setup the parameters optimized for working on individual images rather than sequences
face_model_params.optimiseForImages();
// Setup the visualization
Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance);
2018-01-09 18:08:39 +01:00
// Initialize the face detector if it has not been initialized yet
if (face_detector == null)
2018-01-09 18:08:39 +01:00
{
face_detector = new FaceDetector();
}
2018-01-10 09:27:16 +01:00
// Initialize the face analyser
face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, false, image_output_size, MaskAligned);
2018-01-10 09:27:16 +01:00
2018-01-13 17:34:03 +01:00
// Loading an image file
var frame = new RawImage(reader.GetNextImage());
var gray_frame = new RawImage(reader.GetCurrentFrameGray());
2018-01-13 17:34:03 +01:00
// For FPS tracking
DateTime? startTime = CurrentTime;
var lastFrameTime = CurrentTime;
2018-01-13 17:34:03 +01:00
// This will be false when the image is not available
2018-01-09 18:08:39 +01:00
while (reader.isOpened())
{
if (!thread_running)
{
break;
2018-01-09 18:08:39 +01:00
}
// Setup recording
RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(false, false,
Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs,
RecordGaze, RecordHOG, RecordTracked, RecordAligned,
reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0);
2018-01-09 18:08:39 +01:00
RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root);
2018-01-09 18:08:39 +01:00
// Detect faces here and return bounding boxes
List<Rect> face_detections = new List<Rect>();
List<double> confidences = new List<double>();
face_detector.DetectFacesHOG(face_detections, gray_frame, confidences);
// For visualization
double progress = reader.GetProgress();
2018-01-09 18:08:39 +01:00
for (int i = 0; i < face_detections.Count; ++i)
2018-01-09 18:08:39 +01:00
{
2018-01-25 09:13:02 +01:00
bool detection_succeeding = landmark_detector.DetectFaceLandmarksInImage(gray_frame, face_detections[i], face_model_params);
2018-01-25 09:13:02 +01:00
var landmarks = landmark_detector.CalculateAllLandmarks();
2018-01-10 09:27:16 +01:00
2018-01-09 18:08:39 +01:00
// Predict action units
2018-01-20 10:09:59 +01:00
var au_preds = face_analyser.PredictStaticAUsAndComputeFeatures(frame, landmarks);
// Predic eye gaze
2018-01-25 09:13:02 +01:00
gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
// Only the final face will contain the details
2018-01-31 09:23:48 +01:00
VisualizeFeatures(frame, visualizer_of, landmarks, landmark_detector.GetVisibilities(), detection_succeeding, i == 0, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress);
// Record an observation
2018-01-28 11:16:20 +01:00
RecordObservation(recorder, visualizer_of.GetVisImage(), detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0);
2018-01-09 18:08:39 +01:00
}
2018-01-13 17:34:03 +01:00
frame = new RawImage(reader.GetNextImage());
gray_frame = new RawImage(reader.GetCurrentFrameGray());
// Do not cary state accross images
2018-01-25 09:13:02 +01:00
landmark_detector.Reset();
face_analyser.Reset();
2018-01-18 18:19:44 +01:00
recorder.Close();
lastFrameTime = CurrentTime;
processing_fps.AddFrame();
2018-01-27 08:46:21 +01:00
2018-01-09 18:08:39 +01:00
// TODO how to report errors from the reader here? exceptions? logging? Problem for future versions?
}
EndMode();
}
2018-01-28 11:16:20 +01:00
private void RecordObservation(RecorderOpenFace recorder, RawImage vis_image, bool success, float fx, float fy, float cx, float cy, double timestamp)
{
2018-01-28 11:16:20 +01:00
recorder.SetObservationTimestamp(timestamp);
2018-01-25 09:13:02 +01:00
double confidence = landmark_detector.GetConfidence();
List<double> pose = new List<double>();
2018-01-25 09:13:02 +01:00
landmark_detector.GetPose(pose, fx, fy, cx, cy);
recorder.SetObservationPose(pose);
2018-01-25 09:13:02 +01:00
List<Tuple<double, double>> landmarks_2D = landmark_detector.CalculateAllLandmarks();
List<Tuple<double, double, double>> landmarks_3D = landmark_detector.Calculate3DLandmarks(fx, fy, cx, cy);
List<double> global_params = landmark_detector.GetRigidParams();
2018-01-28 11:16:20 +01:00
List<double> local_params = landmark_detector.GetNonRigidParams();
recorder.SetObservationLandmarks(landmarks_2D, landmarks_3D, global_params, local_params, confidence, success);
var gaze = gaze_analyser.GetGazeCamera();
var gaze_angle = gaze_analyser.GetGazeAngle();
2018-01-25 09:13:02 +01:00
var landmarks_2d_eyes = landmark_detector.CalculateAllEyeLandmarks();
var landmarks_3d_eyes = landmark_detector.CalculateAllEyeLandmarks3D(fx, fy, cx, cy);
recorder.SetObservationGaze(gaze.Item1, gaze.Item2, gaze_angle, landmarks_2d_eyes, landmarks_3d_eyes);
2018-01-18 16:47:58 +01:00
var au_regs = face_analyser.GetCurrentAUsReg();
var au_classes = face_analyser.GetCurrentAUsClass();
recorder.SetObservationActionUnits(au_regs, au_classes);
2018-01-18 17:34:31 +01:00
recorder.SetObservationFaceAlign(face_analyser.GetLatestAlignedFace());
2018-01-19 09:10:34 +01:00
var hog_feature = face_analyser.GetLatestHOGFeature();
2018-01-19 09:10:34 +01:00
recorder.SetObservationHOG(success, hog_feature, face_analyser.GetHOGRows(), face_analyser.GetHOGCols(), face_analyser.GetHOGChannels());
recorder.SetObservationVisualization(vis_image);
2018-01-18 17:34:31 +01:00
recorder.WriteObservation();
}
2018-01-31 09:23:48 +01:00
private void VisualizeFeatures(RawImage frame, Visualizer visualizer, List<Tuple<double, double>> landmarks, List<bool> visibilities, bool detection_succeeding,
bool new_image, float fx, float fy, float cx, float cy, double progress)
{
List<Tuple<Point, Point>> lines = null;
List<Tuple<double, double>> eye_landmarks = null;
List<Tuple<Point, Point>> gaze_lines = null;
Tuple<double, double> gaze_angle = new Tuple<double, double>(0, 0);
List<double> pose = new List<double>();
2018-01-25 09:13:02 +01:00
landmark_detector.GetPose(pose, fx, fy, cx, cy);
List<double> non_rigid_params = landmark_detector.GetNonRigidParams();
2018-01-25 09:13:02 +01:00
double confidence = landmark_detector.GetConfidence();
if (confidence < 0)
confidence = 0;
else if (confidence > 1)
confidence = 1;
2018-01-31 09:23:48 +01:00
double scale = landmark_detector.GetRigidParams()[0];
2017-01-09 22:11:19 +01:00
// Helps with recording and showing the visualizations
2018-01-31 09:23:48 +01:00
if (new_image)
2018-01-21 11:08:24 +01:00
{
visualizer.SetImage(frame, fx, fy, cx, cy);
}
visualizer.SetObservationHOG(face_analyser.GetLatestHOGFeature(), face_analyser.GetHOGRows(), face_analyser.GetHOGCols());
2018-01-31 09:23:48 +01:00
visualizer.SetObservationLandmarks(landmarks, confidence, visibilities);
visualizer.SetObservationPose(pose, confidence);
2018-01-25 09:13:02 +01:00
visualizer.SetObservationGaze(gaze_analyser.GetGazeCamera().Item1, gaze_analyser.GetGazeCamera().Item2, landmark_detector.CalculateAllEyeLandmarks(), landmark_detector.CalculateAllEyeLandmarks3D(fx, fy, cx, cy), confidence);
if (detection_succeeding)
2018-01-31 09:23:48 +01:00
{
2018-01-25 09:13:02 +01:00
eye_landmarks = landmark_detector.CalculateVisibleEyeLandmarks();
lines = landmark_detector.CalculateBox(fx, fy, cx, cy);
2018-01-31 09:23:48 +01:00
gaze_lines = gaze_analyser.CalculateGazeLines(fx, fy, cx, cy);
gaze_angle = gaze_analyser.GetGazeAngle();
}
// Visualisation (as a separate function)
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
if (ShowAUs)
{
var au_classes = face_analyser.GetCurrentAUsClass();
var au_regs = face_analyser.GetCurrentAUsReg();
auClassGraph.Update(au_classes);
var au_regs_scaled = new Dictionary<String, double>();
foreach (var au_reg in au_regs)
{
au_regs_scaled[au_reg.Key] = au_reg.Value / 5.0;
if (au_regs_scaled[au_reg.Key] < 0)
au_regs_scaled[au_reg.Key] = 0;
if (au_regs_scaled[au_reg.Key] > 1)
au_regs_scaled[au_reg.Key] = 1;
}
auRegGraph.Update(au_regs_scaled);
}
if (ShowGeometry)
{
int yaw = (int)(pose[4] * 180 / Math.PI + 0.5);
int roll = (int)(pose[5] * 180 / Math.PI + 0.5);
int pitch = (int)(pose[3] * 180 / Math.PI + 0.5);
YawLabel.Content = yaw + "°";
RollLabel.Content = roll + "°";
PitchLabel.Content = pitch + "°";
XPoseLabel.Content = (int)pose[0] + " mm";
YPoseLabel.Content = (int)pose[1] + " mm";
ZPoseLabel.Content = (int)pose[2] + " mm";
nonRigidGraph.Update(non_rigid_params);
// Update eye gaze
String x_angle = String.Format("{0:F0}°", gaze_angle.Item1 * (180.0 / Math.PI));
String y_angle = String.Format("{0:F0}°", gaze_angle.Item2 * (180.0 / Math.PI));
GazeXLabel.Content = x_angle;
GazeYLabel.Content = y_angle;
}
if (ShowTrackedVideo)
{
2018-01-31 09:23:48 +01:00
if (new_image)
{
latest_img = frame.CreateWriteableBitmap();
}
frame.UpdateWriteableBitmap(latest_img);
2018-01-31 09:23:48 +01:00
overlay_image.Source = latest_img;
overlay_image.Confidence = confidence;
overlay_image.FPS = processing_fps.GetFPS();
overlay_image.Progress = progress;
overlay_image.FaceScale = scale;
if (!detection_succeeding)
{
2018-01-31 09:23:48 +01:00
overlay_image.OverlayLines.Clear();
overlay_image.OverlayPoints.Clear();
overlay_image.OverlayPointsVisibility.Clear();
overlay_image.OverlayEyePoints.Clear();
overlay_image.GazeLines.Clear();
}
else
{
List<Point> landmark_points = new List<Point>();
foreach (var p in landmarks)
{
landmark_points.Add(new Point(p.Item1, p.Item2));
}
List<Point> eye_landmark_points = new List<Point>();
foreach (var p in eye_landmarks)
{
eye_landmark_points.Add(new Point(p.Item1, p.Item2));
}
if (new_image)
{
2018-01-31 09:23:48 +01:00
overlay_image.OverlayLines = lines;
overlay_image.OverlayPoints = landmark_points;
overlay_image.OverlayPointsVisibility = visibilities;
overlay_image.OverlayEyePoints = eye_landmark_points;
overlay_image.GazeLines = gaze_lines;
}
else
{
// In case of multiple faces just add them to the existing drawing list
2018-01-31 09:23:48 +01:00
overlay_image.OverlayLines.AddRange(lines.GetRange(0, lines.Count));
overlay_image.OverlayPoints.AddRange(landmark_points.GetRange(0, landmark_points.Count));
overlay_image.OverlayPointsVisibility.AddRange(visibilities.GetRange(0, visibilities.Count));
overlay_image.OverlayEyePoints.AddRange(eye_landmark_points.GetRange(0, eye_landmark_points.Count));
overlay_image.GazeLines.AddRange(gaze_lines.GetRange(0, gaze_lines.Count));
}
}
}
if (ShowAppearance)
{
RawImage aligned_face = face_analyser.GetLatestAlignedFace();
2018-01-19 17:33:17 +01:00
RawImage hog_face = visualizer.GetHOGVis();
if (latest_aligned_face == null)
{
latest_aligned_face = aligned_face.CreateWriteableBitmap();
latest_HOG_descriptor = hog_face.CreateWriteableBitmap();
}
aligned_face.UpdateWriteableBitmap(latest_aligned_face);
hog_face.UpdateWriteableBitmap(latest_HOG_descriptor);
AlignedFace.Source = latest_aligned_face;
AlignedHOG.Source = latest_HOG_descriptor;
}
}));
}
2016-05-20 22:48:43 +02:00
private void StopTracking()
{
// First complete the running of the thread
if (processing_thread != null)
{
// Tell the other thread to finish
thread_running = false;
processing_thread.Join();
}
}
// ----------------------------------------------------------
// Mode handling (image, video)
// ----------------------------------------------------------
// Disable GUI components that should not be active during processing
private void SetupFeatureExtractionMode()
{
Dispatcher.Invoke((Action)(() =>
{
SettingsMenu.IsEnabled = false;
RecordingMenu.IsEnabled = false;
AUSetting.IsEnabled = false;
PauseButton.IsEnabled = true;
StopButton.IsEnabled = true;
NextFiveFramesButton.IsEnabled = false;
NextFrameButton.IsEnabled = false;
}));
}
// When the processing is done re-enable the components
private void EndMode()
{
latest_img = null;
skip_frames = 0;
// Unpause if it's paused
if (thread_paused)
{
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
PauseButton_Click(null, null);
}));
}
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 1, 0), (Action)(() =>
{
SettingsMenu.IsEnabled = true;
RecordingMenu.IsEnabled = true;
AUSetting.IsEnabled = true;
PauseButton.IsEnabled = false;
StopButton.IsEnabled = false;
NextFiveFramesButton.IsEnabled = false;
NextFrameButton.IsEnabled = false;
// Clean up the interface itself
2018-01-31 09:23:48 +01:00
overlay_image.Source = null;
auClassGraph.Update(new Dictionary<string, double>());
auRegGraph.Update(new Dictionary<string, double>());
YawLabel.Content = "0°";
RollLabel.Content = "0°";
PitchLabel.Content = "0°";
XPoseLabel.Content = "0 mm";
YPoseLabel.Content = "0 mm";
ZPoseLabel.Content = "0 mm";
nonRigidGraph.Update(new List<double>());
GazeXLabel.Content = "0°";
GazeYLabel.Content = "0°";
AlignedFace.Source = null;
AlignedHOG.Source = null;
}));
}
2016-05-20 22:48:43 +02:00
// ----------------------------------------------------------
// Opening Videos/Images
// ----------------------------------------------------------
2018-01-09 18:08:39 +01:00
// Some utilities for opening images/videos and directories
private List<string> openMediaDialog(bool images)
2018-01-09 18:08:39 +01:00
{
string[] image_files = new string[0];
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 2, 0), (Action)(() =>
{
var d = new Microsoft.Win32.OpenFileDialog();
2018-01-09 18:08:39 +01:00
d.Multiselect = true;
if (images)
{
2018-01-09 18:08:39 +01:00
d.Filter = "Image files|*.jpg;*.jpeg;*.bmp;*.png;*.gif";
}
else
{
d.Filter = "Video files|*.avi;*.wmv;*.mov;*.mpg;*.mpeg;*.mp4";
}
if (d.ShowDialog(this) == true)
{
2016-05-20 22:48:43 +02:00
2018-01-09 18:08:39 +01:00
image_files = d.FileNames;
}
}));
List<string> img_files_list = new List<string>(image_files);
return img_files_list;
2018-01-09 18:08:39 +01:00
}
private string openDirectory()
{
string to_return = "";
using (var fbd = new FolderBrowserDialog())
{
DialogResult result = fbd.ShowDialog();
2018-01-27 08:46:21 +01:00
if (result == System.Windows.Forms.DialogResult.OK)
{
to_return = fbd.SelectedPath;
}
2018-01-27 08:46:21 +01:00
else if(!string.IsNullOrWhiteSpace(fbd.SelectedPath))
{
string messageBoxText = "Could not open the directory.";
string caption = "Invalid directory";
MessageBoxButton button = MessageBoxButton.OK;
MessageBoxImage icon = MessageBoxImage.Warning;
// Display message box
System.Windows.MessageBox.Show(messageBoxText, caption, button, icon);
2018-01-09 18:08:39 +01:00
}
}
return to_return;
}
private void imageSequenceFileOpenClick(object sender, RoutedEventArgs e)
{
// First clean up existing tracking
StopTracking();
string directory = openDirectory();
if (!string.IsNullOrWhiteSpace(directory))
{
SequenceReader reader = new SequenceReader(directory, true, fx, fy, cx, cy);
processing_thread = new Thread(() => ProcessSequence(reader));
processing_thread.Name = "Image sequence processing";
processing_thread.Start();
}
}
private void videoFileOpenClick(object sender, RoutedEventArgs e)
{
// First clean up existing tracking
StopTracking();
var video_files = openMediaDialog(false);
processing_thread = new Thread(() => ProcessSequences(video_files));
processing_thread.Name = "Video processing";
processing_thread.Start();
}
// Selecting one or more images in a directory
2018-01-09 18:08:39 +01:00
private void individualImageFilesOpenClick(object sender, RoutedEventArgs e)
{
// First clean up existing tracking
StopTracking();
var image_files = openMediaDialog(true);
ImageReader reader = new ImageReader(image_files, fx, fy, cx, cy);
2018-01-09 18:08:39 +01:00
processing_thread = new Thread(() => ProcessIndividualImages(reader));
processing_thread.Start();
}
// Selecting a directory containing images
2018-01-09 18:08:39 +01:00
private void individualImageDirectoryOpenClick(object sender, RoutedEventArgs e)
2016-05-20 22:48:43 +02:00
{
2018-01-09 18:08:39 +01:00
// First clean up existing tracking
2016-05-20 22:48:43 +02:00
StopTracking();
string directory = openDirectory();
if(!string.IsNullOrWhiteSpace(directory))
{
ImageReader reader = new ImageReader(directory, fx, fy, cx, cy);
2016-05-20 22:48:43 +02:00
processing_thread = new Thread(() => ProcessIndividualImages(reader));
processing_thread.Start();
}
2016-05-20 22:48:43 +02:00
}
private void openWebcamClick(object sender, RoutedEventArgs e)
{
StopTracking();
2018-01-27 08:46:21 +01:00
// If camera selection has already been done, no need to re-populate the list as it is quite slow
if (cam_sec == null)
{
cam_sec = new CameraSelection();
}
else
{
2018-01-27 08:46:21 +01:00
cam_sec = new CameraSelection(cam_sec.cams);
cam_sec.Visibility = System.Windows.Visibility.Visible;
}
2018-01-27 08:46:21 +01:00
// Set the icon
Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);
cam_sec.Icon = BitmapFrame.Create(iconUri);
2018-01-27 08:46:21 +01:00
if (!cam_sec.no_cameras_found)
cam_sec.ShowDialog();
2018-01-27 08:46:21 +01:00
if (cam_sec.camera_selected)
{
int cam_id = cam_sec.selected_camera.Item1;
int width = cam_sec.selected_camera.Item2;
int height = cam_sec.selected_camera.Item3;
SequenceReader reader = new SequenceReader(cam_id, width, height, fx, fy, cx, cy);
2018-01-27 08:46:21 +01:00
processing_thread = new Thread(() => ProcessSequence(reader));
processing_thread.Name = "Webcam processing";
processing_thread.Start();
2018-01-27 08:46:21 +01:00
}
}
2016-05-20 22:48:43 +02:00
// --------------------------------------------------------
// Button handling
// --------------------------------------------------------
// Cleanup stuff when closing the window
private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e)
{
if (processing_thread != null)
{
// Stop capture and tracking
thread_running = false;
processing_thread.Join();
}
}
// Stopping the tracking
private void StopButton_Click(object sender, RoutedEventArgs e)
{
if (processing_thread != null)
{
// Stop capture and tracking
thread_paused = false;
thread_running = false;
// Let the processing thread finish
2016-05-20 22:48:43 +02:00
processing_thread.Join();
// Clean up the interface
EndMode();
2016-05-20 22:48:43 +02:00
}
}
private void PauseButton_Click(object sender, RoutedEventArgs e)
{
if (processing_thread != null)
{
// Stop capture and tracking
thread_paused = !thread_paused;
NextFrameButton.IsEnabled = thread_paused;
NextFiveFramesButton.IsEnabled = thread_paused;
if (thread_paused)
{
PauseButton.Content = "Resume";
}
else
{
PauseButton.Content = "Pause";
}
}
}
private void SkipButton_Click(object sender, RoutedEventArgs e)
{
if (sender.Equals(NextFrameButton))
{
skip_frames += 1;
}
else if (sender.Equals(NextFiveFramesButton))
{
skip_frames += 5;
}
}
private void VisualisationChange(object sender, RoutedEventArgs e)
2016-05-20 22:48:43 +02:00
{
// Collapsing or restoring the windows here
if (!ShowTrackedVideo)
2016-05-20 22:48:43 +02:00
{
VideoBorder.Visibility = System.Windows.Visibility.Collapsed;
MainGrid.ColumnDefinitions[0].Width = new GridLength(0, GridUnitType.Star);
}
else
{
VideoBorder.Visibility = System.Windows.Visibility.Visible;
MainGrid.ColumnDefinitions[0].Width = new GridLength(2.1, GridUnitType.Star);
}
if (!ShowAppearance)
2016-05-20 22:48:43 +02:00
{
AppearanceBorder.Visibility = System.Windows.Visibility.Collapsed;
MainGrid.ColumnDefinitions[1].Width = new GridLength(0, GridUnitType.Star);
}
else
{
AppearanceBorder.Visibility = System.Windows.Visibility.Visible;
MainGrid.ColumnDefinitions[1].Width = new GridLength(0.8, GridUnitType.Star);
}
// Collapsing or restoring the windows here
if (!ShowGeometry)
2016-05-20 22:48:43 +02:00
{
GeometryBorder.Visibility = System.Windows.Visibility.Collapsed;
MainGrid.ColumnDefinitions[2].Width = new GridLength(0, GridUnitType.Star);
}
else
{
GeometryBorder.Visibility = System.Windows.Visibility.Visible;
MainGrid.ColumnDefinitions[2].Width = new GridLength(1.0, GridUnitType.Star);
}
// Collapsing or restoring the windows here
if (!ShowAUs)
2016-05-20 22:48:43 +02:00
{
ActionUnitBorder.Visibility = System.Windows.Visibility.Collapsed;
MainGrid.ColumnDefinitions[3].Width = new GridLength(0, GridUnitType.Star);
}
else
{
ActionUnitBorder.Visibility = System.Windows.Visibility.Visible;
MainGrid.ColumnDefinitions[3].Width = new GridLength(1.6, GridUnitType.Star);
}
}
2018-01-10 09:27:16 +01:00
private void setOutputImageSize_Click(object sender, RoutedEventArgs e)
{
NumberEntryWindow number_entry_window = new NumberEntryWindow(image_output_size);
number_entry_window.Icon = this.Icon;
number_entry_window.WindowStartupLocation = WindowStartupLocation.CenterScreen;
if (number_entry_window.ShowDialog() == true)
{
image_output_size = number_entry_window.OutputInt;
}
}
private void setCameraParameters_Click(object sender, RoutedEventArgs e)
{
CameraParametersEntry camera_params_entry_window = new CameraParametersEntry(fx, fy, cx, cy);
camera_params_entry_window.Icon = this.Icon;
camera_params_entry_window.WindowStartupLocation = WindowStartupLocation.CenterScreen;
if (camera_params_entry_window.ShowDialog() == true)
{
fx = camera_params_entry_window.Fx;
fy = camera_params_entry_window.Fy;
cx = camera_params_entry_window.Cx;
cy = camera_params_entry_window.Cy;
}
}
private void OutputLocationItem_Click(object sender, RoutedEventArgs e)
{
var dlg = new CommonOpenFileDialog();
dlg.Title = "Select output directory";
dlg.IsFolderPicker = true;
dlg.AllowNonFileSystemItems = false;
dlg.EnsureFileExists = true;
dlg.EnsurePathExists = true;
dlg.EnsureReadOnly = false;
dlg.EnsureValidNames = true;
dlg.Multiselect = false;
dlg.ShowPlacesList = true;
if (dlg.ShowDialog() == CommonFileDialogResult.Ok)
{
var folder = dlg.FileName;
record_root = folder;
}
}
2016-05-20 22:48:43 +02:00
}
}