diff --git a/.gitignore b/.gitignore index 893c021..cd86514 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,4 @@ lib/local/CppInerop/Release/ lib/local/CppInerop/x64/ lib/local/FaceAnalyser/Release/ lib/local/LandmarkDetector/Release/ +gui/HeadPose-live/obj/ diff --git a/OpenFace.sln b/OpenFace.sln index 5895f19..6fc60b8 100644 --- a/OpenFace.sln +++ b/OpenFace.sln @@ -33,6 +33,10 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CamCom", "lib\local\CamCom\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenFaceOffline", "gui\OpenFaceOffline\OpenFaceOffline.csproj", "{A4760F41-2B1F-4144-B7B2-62785AFFE79B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenFaceDemo", "gui\OpenFaceDemo\OpenFaceDemo.csproj", "{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeadPoseLive", "gui\HeadPose-live\HeadPoseLive.csproj", "{F396362D-821E-4EA6-9BBF-1F6050844118}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -129,6 +133,22 @@ Global {A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Release|Win32.Build.0 = Release|x86 {A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Release|x64.ActiveCfg = Release|Any CPU {A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Release|x64.Build.0 = Release|Any CPU + {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Debug|Win32.ActiveCfg = Debug|x86 + {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Debug|Win32.Build.0 = Debug|x86 + {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Debug|x64.ActiveCfg = Debug|Any CPU + {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Debug|x64.Build.0 = Debug|Any CPU + {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Release|Win32.ActiveCfg = Release|x86 + {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Release|Win32.Build.0 = Release|x86 + {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Release|x64.ActiveCfg = Release|Any CPU + {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Release|x64.Build.0 = Release|Any CPU + {F396362D-821E-4EA6-9BBF-1F6050844118}.Debug|Win32.ActiveCfg = Debug|x86 + {F396362D-821E-4EA6-9BBF-1F6050844118}.Debug|Win32.Build.0 = Debug|x86 + {F396362D-821E-4EA6-9BBF-1F6050844118}.Debug|x64.ActiveCfg = Debug|Any CPU + {F396362D-821E-4EA6-9BBF-1F6050844118}.Debug|x64.Build.0 = Debug|Any CPU + {F396362D-821E-4EA6-9BBF-1F6050844118}.Release|Win32.ActiveCfg = Release|Any CPU + {F396362D-821E-4EA6-9BBF-1F6050844118}.Release|Win32.Build.0 = Release|Any CPU + {F396362D-821E-4EA6-9BBF-1F6050844118}.Release|x64.ActiveCfg = Release|Any CPU + {F396362D-821E-4EA6-9BBF-1F6050844118}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -145,5 +165,7 @@ Global {78196985-EE54-411F-822B-5A23EDF80642} = {D5F7B3E2-01FE-4F4A-8CE1-274D74D395D4} {0CEC6A75-17BA-4DC5-9405-C74154921E60} = {D5F7B3E2-01FE-4F4A-8CE1-274D74D395D4} {A4760F41-2B1F-4144-B7B2-62785AFFE79B} = {C9D8D0B0-11EC-41CB-8524-2DDB5BE94297} + {E143A2AA-312E-4DFE-B61D-9A87CCBC8E90} = {C9D8D0B0-11EC-41CB-8524-2DDB5BE94297} + {F396362D-821E-4EA6-9BBF-1F6050844118} = {C9D8D0B0-11EC-41CB-8524-2DDB5BE94297} EndGlobalSection EndGlobal diff --git a/gui/HeadPose-live/App.config b/gui/HeadPose-live/App.config new file mode 100644 index 0000000..88fa402 --- /dev/null +++ b/gui/HeadPose-live/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/gui/HeadPose-live/App.xaml b/gui/HeadPose-live/App.xaml new file mode 100644 index 0000000..b58dc6f --- /dev/null +++ b/gui/HeadPose-live/App.xaml @@ -0,0 +1,9 @@ + + + + + diff --git a/gui/HeadPose-live/App.xaml.cs b/gui/HeadPose-live/App.xaml.cs new file mode 100644 index 0000000..f36f6f5 --- /dev/null +++ b/gui/HeadPose-live/App.xaml.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Data; +using System.Linq; +using System.Threading.Tasks; +using System.Windows; + +namespace HeadPose_live +{ + /// + /// Interaction logic for App.xaml + /// + public partial class App : Application + { + } +} diff --git a/gui/HeadPose-live/HeadPoseLive.csproj b/gui/HeadPose-live/HeadPoseLive.csproj new file mode 100644 index 0000000..0565fb4 --- /dev/null +++ b/gui/HeadPose-live/HeadPoseLive.csproj @@ -0,0 +1,179 @@ + + + + + Debug + AnyCPU + {F396362D-821E-4EA6-9BBF-1F6050844118} + WinExe + Properties + HeadPoseLive + HeadPoseLive + v4.5.2 + 512 + {60dc8134-eba5-43b8-bcc9-bb4bc16c2548};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + 4 + true + + + + + x64 + true + full + false + ..\..\x64\Debug\ + DEBUG;TRACE + prompt + 4 + + + x64 + pdbonly + true + ..\..\x64\Release\ + TRACE + prompt + 4 + + + true + ..\..\Debug\ + DEBUG;TRACE + full + x86 + prompt + MinimumRecommendedRules.ruleset + true + + + ..\..\Release\ + TRACE + true + pdbonly + x86 + prompt + MinimumRecommendedRules.ruleset + true + + + logo1.ico + + + + + + + + + + + + + 4.0 + + + + + + ..\..\packages\ZeroMQ.4.1.0.22\lib\net40\ZeroMQ.dll + True + + + + + MSBuild:Compile + Designer + + + Liability.xaml + + + TextEntryWindow.xaml + + + Designer + MSBuild:Compile + + + MSBuild:Compile + Designer + + + App.xaml + Code + + + MainWindow.xaml + Code + + + Designer + MSBuild:Compile + + + + + Code + + + True + True + Resources.resx + + + True + Settings.settings + True + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + SettingsSingleFileGenerator + Settings.Designer.cs + + + + + + + + + {78196985-ee54-411f-822b-5a23edf80642} + CppInerop + + + {e143a2aa-312e-4dfe-b61d-9a87ccbc8e90} + OpenFaceDemo + + + {a4760f41-2b1f-4144-b7b2-62785affe79b} + OpenFaceOffline + + + + + + + + xcopy /I /E /Y /D "$(ProjectDir)logo1.ico" "$(ProjectDir)$(OutDir)" +xcopy /I /E /Y /D "$(ProjectDir)logo1.png" "$(ProjectDir)$(OutDir)" + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + \ No newline at end of file diff --git a/gui/HeadPose-live/Liability.xaml b/gui/HeadPose-live/Liability.xaml new file mode 100644 index 0000000..b770c7f --- /dev/null +++ b/gui/HeadPose-live/Liability.xaml @@ -0,0 +1,26 @@ + + + + + + + This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are disclaimed. +In no event shall copyright holders or contributors be liable for any direct, +indirect, incidental, special, exemplary, or consequential damages +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. + + + + + diff --git a/gui/HeadPose-live/Liability.xaml.cs b/gui/HeadPose-live/Liability.xaml.cs new file mode 100644 index 0000000..9641456 --- /dev/null +++ b/gui/HeadPose-live/Liability.xaml.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Data; +using System.Windows.Documents; +using System.Windows.Input; +using System.Windows.Media; +using System.Windows.Media.Imaging; +using System.Windows.Shapes; + +namespace HeadPoseLive +{ + + /// + /// Interaction logic for Liability.xaml + /// + public partial class Liability : Window + { + public bool continue_pressed = false; + + public Liability() + { + InitializeComponent(); + + this.KeyDown += new KeyEventHandler(TextEntry_KeyDown); + FocusManager.SetFocusedElement(this, ContinueButton); + } + + private void Button_Click(object sender, RoutedEventArgs e) + { + continue_pressed = true; + this.Close(); + } + + private void TextEntry_KeyDown(object sender, KeyEventArgs e) + { + if (e.Key == Key.Enter) + { + continue_pressed = true; + DialogResult = true; + } + } + + } +} diff --git a/gui/HeadPose-live/MainWindow.xaml b/gui/HeadPose-live/MainWindow.xaml new file mode 100644 index 0000000..a61c6f9 --- /dev/null +++ b/gui/HeadPose-live/MainWindow.xaml @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/gui/HeadPose-live/MainWindow.xaml.cs b/gui/HeadPose-live/MainWindow.xaml.cs new file mode 100644 index 0000000..cca6a73 --- /dev/null +++ b/gui/HeadPose-live/MainWindow.xaml.cs @@ -0,0 +1,747 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing.Imaging; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Data; +using System.Windows.Documents; +using System.Windows.Input; +using System.Windows.Media; +using System.Windows.Media.Imaging; +using System.Windows.Navigation; +using System.Windows.Shapes; +using Microsoft.Win32; + +using OpenCVWrappers; +using CppInterop; +using CppInterop.LandmarkDetector; +using CameraInterop; +using FaceAnalyser_Interop; +using System.Windows.Threading; +using FaceAnalyser_Interop; + +using ZeroMQ; +using System.Drawing; +using System.Collections.Concurrent; + +namespace HeadPoseLive +{ + /// + /// Interaction logic for MainWindow.xaml + /// + public partial class MainWindow : Window + { + // Timing for measuring FPS + #region High-Resolution Timing + static DateTime startTime; + static Stopwatch sw = new Stopwatch(); + + static MainWindow() + { + startTime = DateTime.Now; + sw.Start(); + } + + public static DateTime CurrentTime + { + get { return startTime + sw.Elapsed; } + } + + #endregion + + OpenFaceOffline.FpsTracker processing_fps = new OpenFaceOffline.FpsTracker(); + + Thread processing_thread; + Thread rec_thread; + + string subject_id; + bool record_video; + bool record_head_pose; + + // Controls if the view should be mirrored or not + volatile bool mirror_image = false; + + // Capturing and displaying the images + OpenFaceOffline.OverlayImage webcam_img; + + // Some members for displaying the results + private CameraInterop.Capture capture; + private WriteableBitmap latest_img; + + // For tracking + double fx = 500, fy = 500, cx = 0, cy = 0; + bool reset = false; + + // For recording + string record_root = "./head_pose_live_recordings/subject"; + string output_root; + private Object recording_lock = new Object(); + int trial_id = 0; + bool recording = false; + int img_width; + int img_height; + + double seconds_to_record = 10; + + System.IO.StreamWriter recording_success_file = null; + + ConcurrentQueue>> recording_objects; + + // For broadcasting the results + ZeroMQ.ZContext zero_mq_context; + ZeroMQ.ZSocket zero_mq_socket; + + volatile bool running = true; + volatile bool pause = false; + + public void StartExperiment() + { + // Inquire more from the user + + // Get the entry dialogue now for the subject ID + trial_id = 0; + TextEntryWindow subject_id_window = new TextEntryWindow(); + subject_id_window.Icon = this.Icon; + + subject_id_window.WindowStartupLocation = WindowStartupLocation.CenterScreen; + + if (subject_id_window.ShowDialog() == true) + { + + subject_id = subject_id_window.ResponseText; + + // Remove trailing spaces and full stops at the end of the folder name + int old_length; + do + { + old_length = subject_id.Length; + subject_id = subject_id.Trim(); + if (subject_id.Length > 0) + { + while (subject_id[subject_id.Length - 1].Equals('.')) + { + subject_id = subject_id.Substring(0, subject_id.Length - 1); + } + } + } while (subject_id.Length != old_length); + + output_root = record_root + subject_id + "/"; + + if (System.IO.Directory.Exists(output_root)) + { + string messageBoxText = "The recording for subject already exists, are you sure you want to continue?"; + string caption = "Directory exists!"; + MessageBoxButton button = MessageBoxButton.YesNo; + MessageBoxImage icon = MessageBoxImage.Warning; + MessageBoxResult result = MessageBox.Show(messageBoxText, caption, button, icon); + if (result == MessageBoxResult.No) + { + this.Close(); + } + + // Else find the latest trial from which to continu + int trial_id_not_found = 0; + while (System.IO.File.Exists(output_root + '/' + "trial_" + trial_id_not_found + ".avi")) + { + trial_id_not_found++; + } + trial_id = trial_id_not_found; + } + + System.IO.Directory.CreateDirectory(output_root); + + record_video = subject_id_window.RecordVideo; + record_head_pose = subject_id_window.RecordHeadPose; + RecordingButton.Content = "Record trial: " + trial_id; + + } + else + { + this.Close(); + } + + } + + public MainWindow() + { + InitializeComponent(); + + DateTime now = DateTime.Now; + + // Set the icon + Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute); + this.Icon = BitmapFrame.Create(iconUri); + + // Warn about the liability + Liability liab = new Liability(); + liab.Icon = BitmapFrame.Create(iconUri); + liab.ShowDialog(); + + if (!liab.continue_pressed) + { + this.Close(); + return; + } + + BitmapImage src = new BitmapImage(); + src.BeginInit(); + src.UriSource = new Uri("logo1.png", UriKind.RelativeOrAbsolute); + src.CacheOption = BitmapCacheOption.OnLoad; + src.EndInit(); + + logoLabel.Source = src; + + // First make the user chooose a webcam + OpenFaceDemo.CameraSelection cam_select = new OpenFaceDemo.CameraSelection(); + cam_select.Icon = BitmapFrame.Create(iconUri); + + if (!cam_select.no_cameras_found) + { + cam_select.ShowDialog(); + } + + if (cam_select.camera_selected) + { + + // Create the capture device + int cam_id = cam_select.selected_camera.Item1; + img_width = cam_select.selected_camera.Item2; + img_height = cam_select.selected_camera.Item3; + + capture = new CameraInterop.Capture(cam_id, img_width, img_height); + + // Set appropriate fx and cx values + fx = fx * (img_width / 640.0); + fy = fy * (img_height / 480.0); + + fx = (fx + fy) / 2.0; + fy = fx; + + cx = img_width / 2.0; + cy = img_height / 2.0; + + if (capture.isOpened()) + { + + // Create the ZeroMQ context for broadcasting the results + zero_mq_context = ZeroMQ.ZContext.Create(); + zero_mq_socket = new ZSocket(zero_mq_context, ZeroMQ.ZSocketType.PUB); + + // Bind on localhost port 5000 + zero_mq_socket.Bind("tcp://127.0.0.1:5000"); + + // Start the tracking now + processing_thread = new Thread(VideoLoop); + processing_thread.Start(); + } + else + { + + string messageBoxText = "Failed to open a webcam"; + string caption = "Webcam failure"; + MessageBoxButton button = MessageBoxButton.OK; + MessageBoxImage icon = MessageBoxImage.Warning; + + // Display message box + MessageBox.Show(messageBoxText, caption, button, icon); + this.Close(); + } + + // Create an overlay image for display purposes + webcam_img = new OpenFaceOffline.OverlayImage(); + + webcam_img.SetValue(Grid.RowProperty, 1); + webcam_img.SetValue(Grid.ColumnProperty, 1); + MainGrid.Children.Add(webcam_img); + + StartExperiment(); + + } + else + { + cam_select.Close(); + this.Close(); + } + + } + + private bool ProcessFrame(CLNF clnf_model, FaceAnalyserManaged face_analyser, FaceModelParameters model_params, RawImage frame, RawImage grayscale_frame, double fx, double fy, double cx, double cy) + { + bool detection_succeeding = clnf_model.DetectLandmarksInVideo(grayscale_frame, model_params); + face_analyser.AddNextFrame(frame, clnf_model, fx, fy, cx, cy, true, false, false); + return detection_succeeding; + + } + + // Capturing and processing the video frame by frame + private void RecordingLoop() + { + // Set up the recording objects first + Thread.CurrentThread.IsBackground = true; + + System.IO.StreamWriter output_head_pose_file = null; + + if (record_head_pose) + { + String filename_poses = output_root + "/trial_" + trial_id + ".poses.txt"; + output_head_pose_file = new System.IO.StreamWriter(filename_poses); + output_head_pose_file.WriteLine("time(ms), success, pose_X(mm), pose_Y(mm), pose_Z(mm), pitch(deg), yaw(deg), roll(deg)"); + } + + VideoWriter video_writer = null; + + if (record_video) + { + double fps = processing_fps.GetFPS(); + String filename_video = output_root + "/trial_" + trial_id + ".avi"; + video_writer = new VideoWriter(filename_video, img_width, img_height, fps, true); + } + + // The timiing should be when the item is captured, but oh well + Stopwatch stopWatch = new Stopwatch(); + stopWatch.Start(); + + while (recording) + { + Tuple> recording_object; + if (recording_objects.TryDequeue(out recording_object)) + { + + if (record_video) + { + video_writer.Write(recording_object.Item1); + } + + if (record_head_pose) + { + String output_pose_line = stopWatch.ElapsedMilliseconds.ToString(); + if (recording_object.Item2) + output_pose_line += ", 1"; + else + output_pose_line += ", 0"; + + for (int i = 0; i < recording_object.Item3.Count; ++i) + { + double num = recording_object.Item3[i]; + if (i > 2) + { + output_pose_line += ", " + num * 180 / Math.PI; + } + else + { + output_pose_line += ", " + num; + } + } + output_head_pose_file.WriteLine(output_pose_line); + } + } + + Thread.Sleep(10); + } + + // Clean up the recording + if (record_head_pose) + { + output_head_pose_file.Close(); + } + } + + // Capturing and processing the video frame by frame + private void VideoLoop() + { + Thread.CurrentThread.IsBackground = true; + + String root = AppDomain.CurrentDomain.BaseDirectory; + FaceModelParameters model_params = new FaceModelParameters(root, false); + CLNF face_model = new CLNF(model_params); + FaceAnalyserManaged face_analyser = new FaceAnalyserManaged(root, false, 112); + + DateTime? startTime = CurrentTime; + + var lastFrameTime = CurrentTime; + + while (running) + { + + ////////////////////////////////////////////// + // CAPTURE FRAME AND DETECT LANDMARKS FOLLOWED BY THE REQUIRED IMAGE PROCESSING + ////////////////////////////////////////////// + + RawImage frame = null; + try + { + frame = capture.GetNextFrame(mirror_image); + } + catch (CameraInterop.CaptureFailedException) + { + break; + } + + lastFrameTime = CurrentTime; + processing_fps.AddFrame(); + + var grayFrame = capture.GetCurrentFrameGray(); + + if (grayFrame == null) + continue; + + bool detectionSucceeding = ProcessFrame(face_model, face_analyser, model_params, frame, grayFrame, fx, fy, cx, cy); + + lock (recording_lock) + { + + if (recording) + { + // Add objects to recording queues + List pose = new List(); + face_model.GetPose(pose, fx, fy, cx, cy); + RawImage image = new RawImage(frame); + recording_objects.Enqueue(new Tuple>(image, detectionSucceeding, pose)); + + } + } + + List> lines = null; + List> eye_landmarks = null; + List landmarks = new List(); + List> gaze_lines = null; + Tuple gaze_angle = new Tuple(0, 0); + double scale = 0; + + if (detectionSucceeding) + { + List> landmarks_doubles = face_model.CalculateLandmarks(); + + foreach (var p in landmarks_doubles) + landmarks.Add(new System.Windows.Point(p.Item1, p.Item2)); + + eye_landmarks = face_model.CalculateEyeLandmarks(); + + scale = face_model.GetRigidParams()[0]; + + gaze_lines = face_analyser.CalculateGazeLines(scale, (float)fx, (float)fy, (float)cx, (float)cy); + gaze_angle = face_analyser.GetGazeAngle(); + + lines = face_model.CalculateBox((float)fx, (float)fy, (float)cx, (float)cy); + } + + if (reset) + { + face_model.Reset(); + face_analyser.Reset(); + reset = false; + } + + // Visualisation updating + try + { + Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() => + { + if (latest_img == null) + latest_img = frame.CreateWriteableBitmap(); + + List pose = new List(); + face_model.GetPose(pose, fx, fy, cx, cy); + + int yaw = (int)(pose[4] * 180 / Math.PI + 0.5); + int yaw_abs = Math.Abs(yaw); + + int roll = (int)(pose[5] * 180 / Math.PI + 0.5); + int roll_abs = Math.Abs(roll); + + int pitch = (int)(pose[3] * 180 / Math.PI + 0.5); + int pitch_abs = Math.Abs(pitch); + + YawLabel.Content = yaw_abs + "°"; + RollLabel.Content = roll_abs + "°"; + PitchLabel.Content = pitch_abs + "°"; + + if (yaw > 0) + YawLabelDir.Content = "Right"; + else if (yaw < 0) + YawLabelDir.Content = "Left"; + else + YawLabelDir.Content = "Straight"; + + if (pitch > 0) + PitchLabelDir.Content = "Down"; + else if (pitch < 0) + PitchLabelDir.Content = "Up"; + else + PitchLabelDir.Content = "Straight"; + + if (roll > 0) + RollLabelDir.Content = "Left"; + else if (roll < 0) + RollLabelDir.Content = "Right"; + else + RollLabelDir.Content = "Straight"; + + XPoseLabel.Content = (int)pose[0] + " mm"; + YPoseLabel.Content = (int)pose[1] + " mm"; + ZPoseLabel.Content = (int)pose[2] + " mm"; + + String x_angle = String.Format("{0:F0}°", gaze_angle.Item1 * (180.0 / Math.PI)); + String y_angle = String.Format("{0:F0}°", gaze_angle.Item2 * (180.0 / Math.PI)); + YawLabelGaze.Content = x_angle; + PitchLabelGaze.Content = y_angle; + + if (gaze_angle.Item1 > 0) + YawLabelGazeDir.Content = "Right"; + else if (gaze_angle.Item1 < 0) + YawLabelGazeDir.Content = "Left"; + else + YawLabelGazeDir.Content = "Straight"; + + if (gaze_angle.Item2 > 0) + PitchLabelGazeDir.Content = "Down"; + else if (gaze_angle.Item2 < 0) + PitchLabelGazeDir.Content = "Up"; + else + PitchLabelGazeDir.Content = "Straight"; + + double confidence = (-face_model.GetConfidence() + 1) / 2.0; + + if (confidence < 0) + confidence = 0; + else if (confidence > 1) + confidence = 1; + + frame.UpdateWriteableBitmap(latest_img); + + webcam_img.Source = latest_img; + webcam_img.Confidence = confidence; + webcam_img.FPS = processing_fps.GetFPS(); + if (!detectionSucceeding) + { + webcam_img.OverlayLines.Clear(); + webcam_img.OverlayPoints.Clear(); + webcam_img.OverlayEyePoints.Clear(); + webcam_img.GazeLines.Clear(); + } + else + { + webcam_img.OverlayLines = lines; + webcam_img.OverlayPoints = landmarks; + webcam_img.FaceScale = scale; + + List eye_landmark_points = new List(); + foreach (var p in eye_landmarks) + { + eye_landmark_points.Add(new System.Windows.Point(p.Item1, p.Item2)); + } + + + webcam_img.OverlayEyePoints = eye_landmark_points; + webcam_img.GazeLines = gaze_lines; + + // Publish the information for other applications + String str_head_pose = String.Format("{0}:{1:F2}, {2:F2}, {3:F2}, {4:F2}, {5:F2}, {6:F2}", "HeadPose", pose[0], pose[1], pose[2], + pose[3] * 180 / Math.PI, pose[4] * 180 / Math.PI, pose[5] * 180 / Math.PI); + + zero_mq_socket.Send(new ZFrame(str_head_pose, Encoding.UTF8)); + + String str_gaze = String.Format("{0}:{1:F2}, {2:F2}", "GazeAngle", gaze_angle.Item1 * (180.0 / Math.PI), gaze_angle.Item2 * (180.0 / Math.PI)); + + zero_mq_socket.Send(new ZFrame(str_gaze, Encoding.UTF8)); + } + })); + + while (running & pause) + { + + Thread.Sleep(10); + } + + } + catch (TaskCanceledException) + { + // Quitting + break; + } + } + System.Console.Out.WriteLine("Thread finished"); + } + + private void startRecordingButton_Click(object sender, RoutedEventArgs e) + { + lock (recording_lock) + { + RecordingButton.IsEnabled = false; + CompleteButton.IsEnabled = false; + PauseButton.IsEnabled = false; + + recording_objects = new ConcurrentQueue>>(); + + recording = true; + + new Thread(() => + { + Thread.CurrentThread.IsBackground = true; + + // Start the recording thread + rec_thread = new Thread(RecordingLoop); + rec_thread.Start(); + + double d = seconds_to_record * 1000; + + Stopwatch stopWatch = new Stopwatch(); + stopWatch.Start(); + + while (d > 1000) + { + + Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() => + { + RecordingButton.Content = ((int)(d / 1000)).ToString() + " seconds remaining"; + })); + + System.Threading.Thread.Sleep(1000); + + d = seconds_to_record * 1000 - stopWatch.ElapsedMilliseconds; + } + + if (d > 0) + { + System.Threading.Thread.Sleep((int)(d)); + } + + recording = false; + + Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() => + { + RecordingButton.Content = "0 seconds remaining"; + })); + + Dispatcher.Invoke(() => + { + lock (recording_lock) + { + + // Wait for the recording thread to finish before enabling + rec_thread.Join(); + + string messageBoxText = "Was the tracking successful?"; + string caption = "Success of tracking"; + MessageBoxButton button = MessageBoxButton.YesNo; + MessageBoxImage icon = MessageBoxImage.Question; + MessageBoxResult result = MessageBox.Show(messageBoxText, caption, button, icon); + + if (recording_success_file == null) + { + recording_success_file = new System.IO.StreamWriter(output_root + "/recording_success.txt", true); + } + + if (result == MessageBoxResult.Yes) + { + recording_success_file.WriteLine('1'); + } + else + { + recording_success_file.WriteLine('0'); + } + recording_success_file.Flush(); + + trial_id++; + RecordingButton.Content = "Record trial: " + trial_id; + RecordingButton.IsEnabled = true; + CompleteButton.IsEnabled = true; + PauseButton.IsEnabled = true; + + } + + }); + }).Start(); + } + } + + private void ResetButton_Click(object sender, RoutedEventArgs e) + { + reset = true; + } + + private void CompleteButton_Click(object sender, RoutedEventArgs e) + { + StartExperiment(); + } + + private void MirrorButton_Click(object sender, RoutedEventArgs e) + { + mirror_image = !mirror_image; + } + + private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e) + { + + // Let it finish recording + recording = false; + if (rec_thread != null) + { + rec_thread.Join(); + } + + // Stop capture and tracking + running = false; + if (processing_thread != null) + { + processing_thread.Join(); + } + + if (capture != null) + capture.Dispose(); + + } + + private void MorrorButton_Click(object sender, RoutedEventArgs e) + { + + } + + private void PauseButton_Click(object sender, RoutedEventArgs e) + { + pause = !pause; + + if (pause) + { + PauseButton.Content = "Resume"; + } + else + { + PauseButton.Content = "Pause"; + } + } + + private void ScreenshotButton_Click(object sender, RoutedEventArgs e) + { + + int Top = (int)this.Top; + int Left = (int)this.Left; + + int Width = (int)this.Width; + int Height = (int)this.Height; + + using (Bitmap bmpScreenCapture = new Bitmap(Width, + Height)) + { + using (System.Drawing.Graphics g = Graphics.FromImage(bmpScreenCapture)) + { + g.CopyFromScreen(Left, + Top, + 0, 0, + bmpScreenCapture.Size, + CopyPixelOperation.SourceCopy); + + // Write out the bitmap here encoded by a time-stamp? + String fname = output_root + DateTime.Now.ToString("yyyy-MMM-dd--HH-mm-ss") + ".png"; + bmpScreenCapture.Save(fname, ImageFormat.Png); + } + } + } + + } +} \ No newline at end of file diff --git a/gui/HeadPose-live/Properties/AssemblyInfo.cs b/gui/HeadPose-live/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..f0d54a0 --- /dev/null +++ b/gui/HeadPose-live/Properties/AssemblyInfo.cs @@ -0,0 +1,55 @@ +using System.Reflection; +using System.Resources; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Windows; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("HeadPose-live")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Microsoft")] +[assembly: AssemblyProduct("HeadPose-live")] +[assembly: AssemblyCopyright("Copyright © Microsoft 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +//In order to begin building localizable applications, set +//CultureYouAreCodingWith in your .csproj file +//inside a . For example, if you are using US english +//in your source files, set the to en-US. Then uncomment +//the NeutralResourceLanguage attribute below. Update the "en-US" in +//the line below to match the UICulture setting in the project file. + +//[assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.Satellite)] + + +[assembly: ThemeInfo( + ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located + //(used if a resource is not found in the page, + // or application resource dictionaries) + ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located + //(used if a resource is not found in the page, + // app, or any theme specific resource dictionaries) +)] + + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/gui/HeadPose-live/Properties/Resources.Designer.cs b/gui/HeadPose-live/Properties/Resources.Designer.cs new file mode 100644 index 0000000..80229ed --- /dev/null +++ b/gui/HeadPose-live/Properties/Resources.Designer.cs @@ -0,0 +1,63 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace HeadPoseLive.Properties { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("HeadPoseLive.Properties.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + } +} diff --git a/gui/HeadPose-live/Properties/Resources.resx b/gui/HeadPose-live/Properties/Resources.resx new file mode 100644 index 0000000..af7dbeb --- /dev/null +++ b/gui/HeadPose-live/Properties/Resources.resx @@ -0,0 +1,117 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/gui/HeadPose-live/Properties/Settings.Designer.cs b/gui/HeadPose-live/Properties/Settings.Designer.cs new file mode 100644 index 0000000..70abedd --- /dev/null +++ b/gui/HeadPose-live/Properties/Settings.Designer.cs @@ -0,0 +1,26 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace HeadPoseLive.Properties { + + + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "14.0.0.0")] + internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase { + + private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings()))); + + public static Settings Default { + get { + return defaultInstance; + } + } + } +} diff --git a/gui/HeadPose-live/Properties/Settings.settings b/gui/HeadPose-live/Properties/Settings.settings new file mode 100644 index 0000000..033d7a5 --- /dev/null +++ b/gui/HeadPose-live/Properties/Settings.settings @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/gui/HeadPose-live/TextEntryWindow.xaml b/gui/HeadPose-live/TextEntryWindow.xaml new file mode 100644 index 0000000..2b330ae --- /dev/null +++ b/gui/HeadPose-live/TextEntryWindow.xaml @@ -0,0 +1,17 @@ + + + + + + + + + + +