sustaining_gazes/gui/HeadPose-live/MainWindow.xaml.cs

748 lines
28 KiB
C#
Raw Normal View History

///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace: an open source facial behavior analysis toolkit
// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency
// in IEEE Winter Conference on Applications of Computer Vision, 2016
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
// Constrained Local Neural Fields for robust facial landmark detection in the wild.
// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency.
// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013.
//
///////////////////////////////////////////////////////////////////////////////
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing.Imaging;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Media.Imaging;
using OpenCVWrappers;
using CppInterop;
using CppInterop.LandmarkDetector;
using System.Windows.Threading;
using GazeAnalyser_Interop;
using ZeroMQ;
using System.Drawing;
using System.Collections.Concurrent;
namespace HeadPoseLive
{
/// <summary>
/// Interaction logic for MainWindow.xaml
/// </summary>
public partial class MainWindow : Window
{
// Timing for measuring FPS
#region High-Resolution Timing
static DateTime startTime;
static Stopwatch sw = new Stopwatch();
static MainWindow()
{
startTime = DateTime.Now;
sw.Start();
}
public static DateTime CurrentTime
{
get { return startTime + sw.Elapsed; }
}
#endregion
OpenFaceOffline.FpsTracker processing_fps = new OpenFaceOffline.FpsTracker();
Thread processing_thread;
Thread rec_thread;
string subject_id;
bool record_video;
bool record_head_pose;
// Controls if the view should be mirrored or not
volatile bool mirror_image = false;
// Capturing and displaying the images
OpenFaceOffline.OverlayImage webcam_img;
// Some members for displaying the results
private WriteableBitmap latest_img;
// For tracking
bool reset = false;
// For recording
string record_root = "./head_pose_live_recordings/subject";
string output_root;
private Object recording_lock = new Object();
int trial_id = 0;
bool recording = false;
int img_width;
int img_height;
double seconds_to_record = 10;
System.IO.StreamWriter recording_success_file = null;
ConcurrentQueue<Tuple<RawImage, bool, List<double>>> recording_objects;
// For broadcasting the results
ZeroMQ.ZContext zero_mq_context;
ZeroMQ.ZSocket zero_mq_socket;
volatile bool running = true;
volatile bool pause = false;
public void StartExperiment()
{
// Inquire more from the user
// Get the entry dialogue now for the subject ID
trial_id = 0;
TextEntryWindow subject_id_window = new TextEntryWindow();
subject_id_window.Icon = this.Icon;
subject_id_window.WindowStartupLocation = WindowStartupLocation.CenterScreen;
if (subject_id_window.ShowDialog() == true)
{
subject_id = subject_id_window.ResponseText;
// Remove trailing spaces and full stops at the end of the folder name
int old_length;
do
{
old_length = subject_id.Length;
subject_id = subject_id.Trim();
if (subject_id.Length > 0)
{
while (subject_id[subject_id.Length - 1].Equals('.'))
{
subject_id = subject_id.Substring(0, subject_id.Length - 1);
}
}
} while (subject_id.Length != old_length);
output_root = record_root + subject_id + "/";
if (System.IO.Directory.Exists(output_root))
{
string messageBoxText = "The recording for subject already exists, are you sure you want to continue?";
string caption = "Directory exists!";
MessageBoxButton button = MessageBoxButton.YesNo;
MessageBoxImage icon = MessageBoxImage.Warning;
MessageBoxResult result = MessageBox.Show(messageBoxText, caption, button, icon);
if (result == MessageBoxResult.No)
{
this.Close();
}
// Else find the latest trial from which to continu
int trial_id_not_found = 0;
while (System.IO.File.Exists(output_root + '/' + "trial_" + trial_id_not_found + ".avi"))
{
trial_id_not_found++;
}
trial_id = trial_id_not_found;
}
System.IO.Directory.CreateDirectory(output_root);
record_video = subject_id_window.RecordVideo;
record_head_pose = subject_id_window.RecordHeadPose;
RecordingButton.Content = "Record trial: " + trial_id;
}
else
{
this.Close();
}
}
public MainWindow()
{
InitializeComponent();
DateTime now = DateTime.Now;
// Set the icon
Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);
this.Icon = BitmapFrame.Create(iconUri);
// Warn about the liability
Liability liab = new Liability();
liab.Icon = BitmapFrame.Create(iconUri);
liab.ShowDialog();
if (!liab.continue_pressed)
{
this.Close();
return;
}
BitmapImage src = new BitmapImage();
src.BeginInit();
src.UriSource = new Uri("logo1.png", UriKind.RelativeOrAbsolute);
src.CacheOption = BitmapCacheOption.OnLoad;
src.EndInit();
logoLabel.Source = src;
// First make the user chooose a webcam
OpenFaceOffline.CameraSelection cam_select = new OpenFaceOffline.CameraSelection();
cam_select.Icon = BitmapFrame.Create(iconUri);
if (!cam_select.no_cameras_found)
{
cam_select.ShowDialog();
}
if (cam_select.camera_selected)
{
// Create the capture device
int cam_id = cam_select.selected_camera.Item1;
img_width = cam_select.selected_camera.Item2;
img_height = cam_select.selected_camera.Item3;
UtilitiesOF.SequenceReader reader = new UtilitiesOF.SequenceReader(cam_id, img_width, img_height);
if (reader.IsOpened())
{
// Create the ZeroMQ context for broadcasting the results
zero_mq_context = ZeroMQ.ZContext.Create();
zero_mq_socket = new ZSocket(zero_mq_context, ZeroMQ.ZSocketType.PUB);
// Bind on localhost port 5000
zero_mq_socket.Bind("tcp://127.0.0.1:5000");
processing_thread = new Thread(() => VideoLoop(reader));
processing_thread.Name = "Webcam processing";
processing_thread.Start();
}
else
{
string messageBoxText = "Failed to open a webcam";
string caption = "Webcam failure";
MessageBoxButton button = MessageBoxButton.OK;
MessageBoxImage icon = MessageBoxImage.Warning;
// Display message box
MessageBox.Show(messageBoxText, caption, button, icon);
this.Close();
}
// Create an overlay image for display purposes
webcam_img = new OpenFaceOffline.OverlayImage();
webcam_img.SetValue(Grid.RowProperty, 1);
webcam_img.SetValue(Grid.ColumnProperty, 1);
MainGrid.Children.Add(webcam_img);
StartExperiment();
}
else
{
cam_select.Close();
this.Close();
}
}
2018-01-25 09:13:02 +01:00
private bool ProcessFrame(CLNF landmark_detector, GazeAnalyserManaged gaze_analyser, FaceModelParameters model_params, RawImage frame, RawImage grayscale_frame, double fx, double fy, double cx, double cy)
{
2018-01-25 09:13:02 +01:00
bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(grayscale_frame, model_params);
gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, fx, fy, cx, cy);
return detection_succeeding;
}
// Capturing and processing the video frame by frame
private void RecordingLoop()
{
// Set up the recording objects first
Thread.CurrentThread.IsBackground = true;
System.IO.StreamWriter output_head_pose_file = null;
if (record_head_pose)
{
String filename_poses = output_root + "/trial_" + trial_id + ".poses.txt";
output_head_pose_file = new System.IO.StreamWriter(filename_poses);
output_head_pose_file.WriteLine("time(ms), success, pose_X(mm), pose_Y(mm), pose_Z(mm), pitch(deg), yaw(deg), roll(deg)");
}
VideoWriter video_writer = null;
if (record_video)
{
double fps = processing_fps.GetFPS();
String filename_video = output_root + "/trial_" + trial_id + ".avi";
video_writer = new VideoWriter(filename_video, img_width, img_height, fps, true);
}
// The timiing should be when the item is captured, but oh well
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
while (recording)
{
Tuple<RawImage, bool, List<double>> recording_object;
if (recording_objects.TryDequeue(out recording_object))
{
if (record_video)
{
video_writer.Write(recording_object.Item1);
}
if (record_head_pose)
{
String output_pose_line = stopWatch.ElapsedMilliseconds.ToString();
if (recording_object.Item2)
output_pose_line += ", 1";
else
output_pose_line += ", 0";
for (int i = 0; i < recording_object.Item3.Count; ++i)
{
double num = recording_object.Item3[i];
if (i > 2)
{
output_pose_line += ", " + num * 180 / Math.PI;
}
else
{
output_pose_line += ", " + num;
}
}
output_head_pose_file.WriteLine(output_pose_line);
}
}
Thread.Sleep(10);
}
// Clean up the recording
if (record_head_pose)
{
output_head_pose_file.Close();
}
}
// Capturing and processing the video frame by frame
private void VideoLoop(UtilitiesOF.SequenceReader reader)
{
Thread.CurrentThread.IsBackground = true;
String root = AppDomain.CurrentDomain.BaseDirectory;
FaceModelParameters model_params = new FaceModelParameters(root, false);
CLNF face_model = new CLNF(model_params);
GazeAnalyserManaged gaze_analyser = new GazeAnalyserManaged();
DateTime? startTime = CurrentTime;
var lastFrameTime = CurrentTime;
while (running)
{
//////////////////////////////////////////////
// CAPTURE FRAME AND DETECT LANDMARKS FOLLOWED BY THE REQUIRED IMAGE PROCESSING
//////////////////////////////////////////////
2018-01-27 12:12:00 +01:00
RawImage frame = reader.GetNextImage();
lastFrameTime = CurrentTime;
processing_fps.AddFrame();
var grayFrame = reader.GetCurrentFrameGray();
2018-01-27 12:12:00 +01:00
if (mirror_image)
{
frame.Mirror();
grayFrame.Mirror();
}
bool detectionSucceeding = ProcessFrame(face_model, gaze_analyser, model_params, frame, grayFrame, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
lock (recording_lock)
{
if (recording)
{
// Add objects to recording queues
List<double> pose = new List<double>();
face_model.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
RawImage image = new RawImage(frame);
recording_objects.Enqueue(new Tuple<RawImage, bool, List<double>>(image, detectionSucceeding, pose));
}
}
List<Tuple<System.Windows.Point, System.Windows.Point>> lines = null;
List<Tuple<double, double>> eye_landmarks = null;
List<System.Windows.Point> landmarks = new List<System.Windows.Point>();
List<Tuple<System.Windows.Point, System.Windows.Point>> gaze_lines = null;
Tuple<double, double> gaze_angle = new Tuple<double, double>(0, 0);
2018-01-31 09:23:48 +01:00
double scale = face_model.GetRigidParams()[0];
if (detectionSucceeding)
{
List<Tuple<double, double>> landmarks_doubles = face_model.CalculateVisibleLandmarks();
foreach (var p in landmarks_doubles)
landmarks.Add(new System.Windows.Point(p.Item1, p.Item2));
eye_landmarks = face_model.CalculateVisibleEyeLandmarks();
2018-01-31 09:23:48 +01:00
gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
gaze_angle = gaze_analyser.GetGazeAngle();
lines = face_model.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
}
if (reset)
{
face_model.Reset();
reset = false;
}
// Visualisation updating
try
{
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
if (latest_img == null)
latest_img = frame.CreateWriteableBitmap();
List<double> pose = new List<double>();
face_model.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
int yaw = (int)(pose[4] * 180 / Math.PI + 0.5);
int yaw_abs = Math.Abs(yaw);
int roll = (int)(pose[5] * 180 / Math.PI + 0.5);
int roll_abs = Math.Abs(roll);
int pitch = (int)(pose[3] * 180 / Math.PI + 0.5);
int pitch_abs = Math.Abs(pitch);
YawLabel.Content = yaw_abs + "°";
RollLabel.Content = roll_abs + "°";
PitchLabel.Content = pitch_abs + "°";
if (yaw > 0)
YawLabelDir.Content = "Right";
else if (yaw < 0)
YawLabelDir.Content = "Left";
else
YawLabelDir.Content = "Straight";
if (pitch > 0)
PitchLabelDir.Content = "Down";
else if (pitch < 0)
PitchLabelDir.Content = "Up";
else
PitchLabelDir.Content = "Straight";
if (roll > 0)
RollLabelDir.Content = "Left";
else if (roll < 0)
RollLabelDir.Content = "Right";
else
RollLabelDir.Content = "Straight";
XPoseLabel.Content = (int)pose[0] + " mm";
YPoseLabel.Content = (int)pose[1] + " mm";
ZPoseLabel.Content = (int)pose[2] + " mm";
String x_angle = String.Format("{0:F0}°", gaze_angle.Item1 * (180.0 / Math.PI));
String y_angle = String.Format("{0:F0}°", gaze_angle.Item2 * (180.0 / Math.PI));
YawLabelGaze.Content = x_angle;
PitchLabelGaze.Content = y_angle;
if (gaze_angle.Item1 > 0)
YawLabelGazeDir.Content = "Right";
else if (gaze_angle.Item1 < 0)
YawLabelGazeDir.Content = "Left";
else
YawLabelGazeDir.Content = "Straight";
if (gaze_angle.Item2 > 0)
PitchLabelGazeDir.Content = "Down";
else if (gaze_angle.Item2 < 0)
PitchLabelGazeDir.Content = "Up";
else
PitchLabelGazeDir.Content = "Straight";
double confidence = face_model.GetConfidence();
if (confidence < 0)
confidence = 0;
else if (confidence > 1)
confidence = 1;
frame.UpdateWriteableBitmap(latest_img);
webcam_img.Source = latest_img;
webcam_img.Confidence = confidence;
webcam_img.FPS = processing_fps.GetFPS();
if (!detectionSucceeding)
{
webcam_img.OverlayLines.Clear();
webcam_img.OverlayPoints.Clear();
webcam_img.OverlayEyePoints.Clear();
webcam_img.GazeLines.Clear();
}
else
{
webcam_img.OverlayLines = lines;
webcam_img.OverlayPoints = landmarks;
webcam_img.FaceScale = scale;
List<System.Windows.Point> eye_landmark_points = new List<System.Windows.Point>();
foreach (var p in eye_landmarks)
{
eye_landmark_points.Add(new System.Windows.Point(p.Item1, p.Item2));
}
webcam_img.OverlayEyePoints = eye_landmark_points;
webcam_img.GazeLines = gaze_lines;
// Publish the information for other applications
String str_head_pose = String.Format("{0}:{1:F2}, {2:F2}, {3:F2}, {4:F2}, {5:F2}, {6:F2}", "HeadPose", pose[0], pose[1], pose[2],
pose[3] * 180 / Math.PI, pose[4] * 180 / Math.PI, pose[5] * 180 / Math.PI);
zero_mq_socket.Send(new ZFrame(str_head_pose, Encoding.UTF8));
String str_gaze = String.Format("{0}:{1:F2}, {2:F2}", "GazeAngle", gaze_angle.Item1 * (180.0 / Math.PI), gaze_angle.Item2 * (180.0 / Math.PI));
zero_mq_socket.Send(new ZFrame(str_gaze, Encoding.UTF8));
}
}));
while (running & pause)
{
Thread.Sleep(10);
}
}
catch (TaskCanceledException)
{
// Quitting
break;
}
}
reader.Close();
System.Console.Out.WriteLine("Thread finished");
}
private void startRecordingButton_Click(object sender, RoutedEventArgs e)
{
lock (recording_lock)
{
RecordingButton.IsEnabled = false;
CompleteButton.IsEnabled = false;
PauseButton.IsEnabled = false;
recording_objects = new ConcurrentQueue<Tuple<RawImage, bool, List<double>>>();
recording = true;
new Thread(() =>
{
Thread.CurrentThread.IsBackground = true;
// Start the recording thread
rec_thread = new Thread(RecordingLoop);
rec_thread.Start();
double d = seconds_to_record * 1000;
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
while (d > 1000)
{
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
RecordingButton.Content = ((int)(d / 1000)).ToString() + " seconds remaining";
}));
System.Threading.Thread.Sleep(1000);
d = seconds_to_record * 1000 - stopWatch.ElapsedMilliseconds;
}
if (d > 0)
{
System.Threading.Thread.Sleep((int)(d));
}
recording = false;
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
RecordingButton.Content = "0 seconds remaining";
}));
Dispatcher.Invoke(() =>
{
lock (recording_lock)
{
// Wait for the recording thread to finish before enabling
rec_thread.Join();
string messageBoxText = "Was the tracking successful?";
string caption = "Success of tracking";
MessageBoxButton button = MessageBoxButton.YesNo;
MessageBoxImage icon = MessageBoxImage.Question;
MessageBoxResult result = MessageBox.Show(messageBoxText, caption, button, icon);
if (recording_success_file == null)
{
recording_success_file = new System.IO.StreamWriter(output_root + "/recording_success.txt", true);
}
if (result == MessageBoxResult.Yes)
{
recording_success_file.WriteLine('1');
}
else
{
recording_success_file.WriteLine('0');
}
recording_success_file.Flush();
trial_id++;
RecordingButton.Content = "Record trial: " + trial_id;
RecordingButton.IsEnabled = true;
CompleteButton.IsEnabled = true;
PauseButton.IsEnabled = true;
}
});
}).Start();
}
}
private void ResetButton_Click(object sender, RoutedEventArgs e)
{
reset = true;
}
private void CompleteButton_Click(object sender, RoutedEventArgs e)
{
StartExperiment();
}
private void MirrorButton_Click(object sender, RoutedEventArgs e)
{
mirror_image = !mirror_image;
}
private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e)
{
// Let it finish recording
recording = false;
if (rec_thread != null)
{
rec_thread.Join();
}
// Stop capture and tracking
running = false;
if (processing_thread != null)
{
processing_thread.Join();
}
}
private void MorrorButton_Click(object sender, RoutedEventArgs e)
{
}
private void PauseButton_Click(object sender, RoutedEventArgs e)
{
pause = !pause;
if (pause)
{
PauseButton.Content = "Resume";
}
else
{
PauseButton.Content = "Pause";
}
}
private void ScreenshotButton_Click(object sender, RoutedEventArgs e)
{
int Top = (int)this.Top;
int Left = (int)this.Left;
int Width = (int)this.Width;
int Height = (int)this.Height;
using (Bitmap bmpScreenCapture = new Bitmap(Width,
Height))
{
using (System.Drawing.Graphics g = Graphics.FromImage(bmpScreenCapture))
{
g.CopyFromScreen(Left,
Top,
0, 0,
bmpScreenCapture.Size,
CopyPixelOperation.SourceCopy);
// Write out the bitmap here encoded by a time-stamp?
String fname = output_root + DateTime.Now.ToString("yyyy-MMM-dd--HH-mm-ss") + ".png";
bmpScreenCapture.Save(fname, ImageFormat.Png);
}
}
}
}
}