Editing camera calibration from the GUI, nicer interface for setting output size

gui/OpenFaceOffline/MainWindow.xaml

@@ -44,8 +44,9 @@
                 <MenuItem Name="OutputLocationItem" Header="Set output location..." Click="OutputLocationItem_Click" ></MenuItem>
                 <MenuItem Header="Set output image size..." Click="setOutputImageSize_Click"></MenuItem>
             </MenuItem>
-            <MenuItem Name="AUSetting" Header="AU settings" >
+            <MenuItem Name="AUSetting" Header="OpenFace settings" >
-                <MenuItem IsCheckable="True" Header="Use dynamic models" Click="UseDynamicModelsCheckBox_Click" IsChecked="{Binding DynamicAUModels}"/>
+                <MenuItem IsCheckable="True" Header="Use dynamic AU models" Click="UseDynamicModelsCheckBox_Click" IsChecked="{Binding DynamicAUModels}"/>
+                <MenuItem Header="Set Camera parameters..." Click="setCameraParameters_Click"/>
             </MenuItem>
             <MenuItem Header="View">
                 <MenuItem IsCheckable="True" Header="Show Video" Click="VisualisationChange" IsChecked="{Binding ShowTrackedVideo}"/>

gui/OpenFaceOffline/MainWindow.xaml.cs

@@ -148,16 +148,15 @@ namespace OpenFaceOffline
 
         int image_output_size = 112;
         
-        // TODO classifiers converted to regressors
-
-        // TODO indication that track is done        
-        
         // Where the recording is done (by default in a record directory, from where the application executed)
         String record_root = "./record";
 
         // For AU prediction, if videos are long dynamic models should be used
         public bool DynamicAUModels { get; set; } = true;
 
+        // Camera calibration parameters
+        public double fx = -1, fy = -1, cx = -1, cy = -1;
+
         public MainWindow()
         {
             InitializeComponent();
@@ -374,15 +373,18 @@ namespace OpenFaceOffline
             clnf_model.Reset();
             face_analyser.Reset();
 
-            // TODO add an ability to change these through a calibration procedure or setting menu
+            // If the camera calibration parameters are not set (indicated by -1), guesstimate them
-            double fx = 500.0 * (capture.width / 640.0);
+            if(fx == -1 || fy == -1 || cx == -1 || cy == -1)
-            double fy = 500.0 * (capture.height / 480.0);
+            { 
+                fx = 500.0 * (capture.width / 640.0);
+                fy = 500.0 * (capture.height / 480.0);
 
-            fx = (fx + fy) / 2.0;
+                fx = (fx + fy) / 2.0;
-            fy = fx;
+                fy = fx;
 
-            double cx = capture.width / 2f;
+                cx = capture.width / 2f;
-            double cy = capture.height / 2f;
+                cy = capture.height / 2f;
+            }
 
             // Setup the recorder first
             recorder = new Recorder(record_root, output_file_name, capture.width, capture.height, Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose,
@@ -410,7 +412,6 @@ namespace OpenFaceOffline
                     break;
                 }
 
-                // TODO stop button should actually clear the video
                 lastFrameTime = CurrentTime;
                 processing_fps.AddFrame();
 
@@ -469,19 +470,6 @@ namespace OpenFaceOffline
 
             var lastFrameTime = CurrentTime;
            
-            clnf_model.Reset();
-            face_analyser.Reset();
-
-            // TODO these need to be stored so that they could be loaded somewhere
-            double fx = 500.0 * (capture.width / 640.0);
-            double fy = 500.0 * (capture.height / 480.0);
-
-            fx = (fx + fy) / 2.0;
-            fy = fx;
-
-            double cx = capture.width / 2f;
-            double cy = capture.height / 2f;
-
             int frame_id = 0;
 
             double fps = capture.GetFPS();
@@ -1024,7 +1012,7 @@ namespace OpenFaceOffline
         private void setOutputImageSize_Click(object sender, RoutedEventArgs e)
         {
 
-            NumberEntryWindow number_entry_window = new NumberEntryWindow();
+            NumberEntryWindow number_entry_window = new NumberEntryWindow(image_output_size);
             number_entry_window.Icon = this.Icon;
 
             number_entry_window.WindowStartupLocation = WindowStartupLocation.CenterScreen;
@@ -1038,6 +1026,22 @@ namespace OpenFaceOffline
             }
         }
 
+        private void setCameraParameters_Click(object sender, RoutedEventArgs e)
+        {
+            CameraParametersEntry camera_params_entry_window = new CameraParametersEntry(fx, fy, cx, cy);
+            camera_params_entry_window.Icon = this.Icon;
+
+            camera_params_entry_window.WindowStartupLocation = WindowStartupLocation.CenterScreen;
+
+            if (camera_params_entry_window.ShowDialog() == true)
+            {
+                fx = camera_params_entry_window.Fx;
+                fy = camera_params_entry_window.Fy;
+                cx = camera_params_entry_window.Cx;
+                cy = camera_params_entry_window.Cy;
+            }
+        }
+
         private void OutputLocationItem_Click(object sender, RoutedEventArgs e)
         {
             var dlg = new CommonOpenFileDialog();
@@ -1057,5 +1061,6 @@ namespace OpenFaceOffline
                 record_root = folder;
             }
         }
+
     }
 }

gui/OpenFaceOffline/OpenFaceOffline.csproj

@@ -94,6 +94,9 @@
     <Compile Include="UI_items\BarGraphHorizontal.xaml.cs">
       <DependentUpon>BarGraphHorizontal.xaml</DependentUpon>
     </Compile>
+    <Compile Include="UI_items\CameraParametersEntry.xaml.cs">
+      <DependentUpon>CameraParametersEntry.xaml</DependentUpon>
+    </Compile>
     <Compile Include="UI_items\MultiBarGraph.xaml.cs">
       <DependentUpon>MultiBarGraph.xaml</DependentUpon>
     </Compile>
@@ -129,6 +132,10 @@
       <SubType>Designer</SubType>
       <Generator>MSBuild:Compile</Generator>
     </Page>
+    <Page Include="UI_items\CameraParametersEntry.xaml">
+      <SubType>Designer</SubType>
+      <Generator>MSBuild:Compile</Generator>
+    </Page>
     <Page Include="UI_items\MultiBarGraph.xaml">
       <SubType>Designer</SubType>
       <Generator>MSBuild:Compile</Generator>

gui/OpenFaceOffline/UI_items/CameraParametersEntry.xaml

@@ -0,0 +1,47 @@
+<Window x:Class="OpenFaceOffline.CameraParametersEntry"
+        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+        xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+        mc:Ignorable="d"
+        Title="Input camera parameters" Height="180" Width="300" ResizeMode="NoResize">
+    <Grid Background="WhiteSmoke">
+        <Grid.ColumnDefinitions>
+            <ColumnDefinition Width="0.7*"/>
+            <ColumnDefinition Width="0.25*"/>
+            <ColumnDefinition Width="0.25*"/>
+            <ColumnDefinition Width="0.25*"/>
+            <ColumnDefinition Width="0.25*"/>
+        </Grid.ColumnDefinitions>
+        <Grid.RowDefinitions>
+            <RowDefinition Height="0.5*"/>
+            <RowDefinition Height="0.5*"/>
+            <RowDefinition Height="0.5*"/>
+            <RowDefinition Height="0.5*"/>
+            <RowDefinition Height="0.5*"/>
+            <RowDefinition Height="0.5*"/>
+        </Grid.RowDefinitions>
+            
+        <TextBlock Grid.Row="0" Grid.Column="0" Grid.ColumnSpan="5" HorizontalAlignment="Left" Text="Enter camera parameters manually:" FontSize="13" VerticalAlignment="Center"/>
+
+        <Label Grid.Row="1" Grid.Column="0">Focal lengths</Label>
+        <Label Grid.Row="1" Grid.Column="1" HorizontalAlignment="Right" VerticalAlignment="Center">fx=</Label>
+        <TextBox Grid.Row="1" Grid.Column="2" VerticalContentAlignment="Center" IsEnabled="False" x:Name="fxTextBox"   />
+        <Label Grid.Row="1" Grid.Column="3" HorizontalAlignment="Right">fy=</Label>
+        <TextBox Grid.Row="1" Grid.Column="4"  VerticalContentAlignment="Center" IsEnabled="False" x:Name="fyTextBox" />
+
+        <Label Grid.Row="2" Grid.Column="0">Optical centre lengths</Label>
+        <Label Grid.Row="2" Grid.Column="1" HorizontalAlignment="Right">cx=</Label>
+        <TextBox Grid.Row="2" Grid.Column="2" VerticalContentAlignment="Center" IsEnabled="False" x:Name="cxTextBox" />
+        <Label Grid.Row="2" Grid.Column="3" HorizontalAlignment="Right">cy=</Label>
+        <TextBox Grid.Row="2" Grid.Column="4" VerticalContentAlignment="Center" IsEnabled="False"  x:Name="cyTextBox"/>
+        
+            <Label Grid.Row="3" Grid.Column="0" Grid.ColumnSpan="5" Name="warningLabel" Visibility="Hidden" FontStyle="Italic" Foreground="Red" HorizontalAlignment="Center">Parameters have to be a non negative reals</Label>
+        <Label Grid.Row="4" Grid.Column="0" Grid.ColumnSpan="4" FontSize="13">Infer parameters automatically</Label>
+        <Separator Grid.Row="4" Grid.Column="0" Grid.ColumnSpan="5" VerticalAlignment="Top"></Separator>
+        <CheckBox Name="automaticCheckBox" Grid.Row="4" Grid.Column="3" HorizontalAlignment="Center" VerticalAlignment="Center" IsChecked="True" Click="CheckBox_Click"></CheckBox>
+        <Button Grid.Row="5" Grid.Column="5" Content="OK" Click="OKButton_Click" Width="40" VerticalAlignment="Bottom" HorizontalAlignment="Right"/>
+
+    </Grid>
+
+</Window>

gui/OpenFaceOffline/UI_items/CameraParametersEntry.xaml.cs

@@ -0,0 +1,219 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge,
+// all rights reserved.
+//
+// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS
+// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS
+// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY.
+// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Notwithstanding the license granted herein, Licensee acknowledges that certain components
+// of the Software may be covered by so-called “open source” software licenses (“Open Source
+// Components”), which means any software licenses approved as open source licenses by the
+// Open Source Initiative or any substantially similar licenses, including without limitation any
+// license that, as a condition of distribution of the software licensed under such license,
+// requires that the distributor make the software available in source code format. Licensor shall
+// provide a list of Open Source Components for a particular version of the Software upon
+// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to
+// the extent required by the licenses covering Open Source Components, the terms of such
+// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the
+// licenses applicable to Open Source Components prohibit any of the restrictions in this
+// License Agreement with respect to such Open Source Component, such restrictions will not
+// apply to such Open Source Component. To the extent the terms of the licenses applicable to
+// Open Source Components require Licensor to make an offer to provide source code or
+// related information in connection with the Software, such offer is hereby made. Any request
+// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk
+// Licensee acknowledges receipt of notices for the Open Source Components for the initial
+// delivery of the Software.
+
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace: an open source facial behavior analysis toolkit
+//       Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency
+//       in IEEE Winter Conference on Applications of Computer Vision, 2016  
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+//       Constrained Local Neural Fields for robust facial landmark detection in the wild.
+//       Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. 
+//       in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013.    
+//
+///////////////////////////////////////////////////////////////////////////////
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Data;
+using System.Windows.Documents;
+using System.Windows.Input;
+using System.Windows.Media;
+using System.Windows.Media.Imaging;
+using System.Windows.Shapes;
+
+namespace OpenFaceOffline
+{
+    /// <summary>
+    /// Interaction logic for TextEntryWindow.xaml
+    /// </summary>
+    public partial class CameraParametersEntry : Window
+    {
+        public CameraParametersEntry(double fx, double fy, double cx, double cy)
+        {
+            InitializeComponent();
+            this.KeyDown += new KeyEventHandler(TextEntry_KeyDown);
+
+            if(fx == -1 || fy == -1 || cx == -1 || cy == -1)
+            {
+                this.fx = 500; this.fy = 500; this.cx = 320; this.cy = 240;
+            }
+            else
+            {
+                this.fx = fx; this.fy = fy; this.cx = cx; this.cy = cy;
+                automaticCheckBox.IsChecked = false;
+                fxTextBox.IsEnabled = true;
+                fyTextBox.IsEnabled = true;
+                cxTextBox.IsEnabled = true;
+                cyTextBox.IsEnabled = true;
+            }
+
+            fxTextBox.Text = this.fx.ToString();
+            fyTextBox.Text = this.fy.ToString();
+            cxTextBox.Text = this.cx.ToString();
+            cyTextBox.Text = this.cy.ToString();
+
+            fxTextBox.TextChanged += ResponseTextBox_TextChanged;
+            fyTextBox.TextChanged += ResponseTextBox_TextChanged;
+            cxTextBox.TextChanged += ResponseTextBox_TextChanged;
+            cyTextBox.TextChanged += ResponseTextBox_TextChanged;
+
+        }
+
+        private void OKButton_Click(object sender, System.Windows.RoutedEventArgs e)
+        {
+            DialogResult = true;
+        }
+
+        private void TextEntry_KeyDown(object sender, KeyEventArgs e)
+        {
+            if (e.Key == Key.Enter)
+            {
+                DialogResult = true;
+            }
+        }
+
+        private double fx = -1, fy = -1, cx = -1, cy = -1;
+
+        public bool IsAutomatic
+        {
+            get { return automaticCheckBox.IsChecked == true; }
+        }
+        public double Fx
+        {
+            get { return automaticCheckBox.IsChecked == true ? -1 : fx; }
+        }
+
+        public double Fy
+        {
+            get { return automaticCheckBox.IsChecked == true ? -1 : fy; }
+        }
+        public double Cx
+        {
+            get { return automaticCheckBox.IsChecked == true ? -1 : cx; }
+        }
+        public double Cy
+        {
+            get { return automaticCheckBox.IsChecked == true ? -1 : cy; }
+        }
+
+        private void CheckBox_Click(object sender, RoutedEventArgs e)
+        {
+            if(automaticCheckBox.IsChecked == true)
+            {
+                fxTextBox.IsEnabled = false;
+                fyTextBox.IsEnabled = false;
+                cxTextBox.IsEnabled = false;
+                cyTextBox.IsEnabled = false;
+            }
+            else
+            {
+                fxTextBox.IsEnabled = true;
+                fyTextBox.IsEnabled = true;
+                cxTextBox.IsEnabled = true;
+                cyTextBox.IsEnabled = true;
+            }
+        }
+
+        // Do not allow illegal characters like
+        private void ResponseTextBox_TextChanged(object sender, TextChangedEventArgs e)
+        {
+            try
+            {
+                double fx_n = Double.Parse(fxTextBox.Text);
+                double fy_n = Double.Parse(fyTextBox.Text);
+                double cx_n = Double.Parse(cxTextBox.Text);
+                double cy_n = Double.Parse(cyTextBox.Text);
+
+                if (fx_n > 0 && fy_n > 0 && cx_n > 0 && cy_n > 0)
+                {
+                    fx = fx_n;
+                    fy = fy_n;
+                    cx = cx_n;
+                    cy = cy_n;
+                    warningLabel.Visibility = System.Windows.Visibility.Hidden;
+                }
+                else
+                {
+                    warningLabel.Visibility = System.Windows.Visibility.Visible;
+
+                    fxTextBox.Text = fx.ToString();
+                    fyTextBox.Text = fy.ToString();
+                    cxTextBox.Text = cx.ToString();
+                    cyTextBox.Text = cy.ToString();
+
+                    fxTextBox.SelectionStart = fxTextBox.Text.Length;
+                    fyTextBox.SelectionStart = fyTextBox.Text.Length;
+                    cxTextBox.SelectionStart = cxTextBox.Text.Length;
+                    cyTextBox.SelectionStart = cyTextBox.Text.Length;
+
+                }
+            }
+            catch (FormatException except)
+            {
+                fxTextBox.Text = fx.ToString();
+                fyTextBox.Text = fy.ToString();
+                cxTextBox.Text = cx.ToString();
+                cyTextBox.Text = cy.ToString();
+
+                fxTextBox.SelectionStart = fxTextBox.Text.Length;
+                fyTextBox.SelectionStart = fyTextBox.Text.Length;
+                cxTextBox.SelectionStart = cxTextBox.Text.Length;
+                cyTextBox.SelectionStart = cyTextBox.Text.Length;
+
+                warningLabel.Visibility = System.Windows.Visibility.Visible;
+            }
+
+        }
+
+    }
+}

gui/OpenFaceOffline/UI_items/NumberEntryWindow.xaml

@@ -4,14 +4,17 @@
         xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
         xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
         mc:Ignorable="d"
-        Title="NumberEntryWindow" Height="160" Width="300">
+        Title="Output image size" Height="110" Width="250" ResizeMode="NoResize">
-    <Grid>
+    <Grid Background="WhiteSmoke">
-        <StackPanel FocusManager.FocusedElement="{Binding ElementName=ResponseTextBox}">
+        <StackPanel>
-            <TextBlock HorizontalAlignment="Center" Text="Enter new output image size" FontSize="20"/>
+            <StackPanel FocusManager.FocusedElement="{Binding ElementName=ResponseTextBox}" Orientation="Horizontal" HorizontalAlignment="Center">
-            <TextBox Margin="0,4,0,0" x:Name="ResponseTextBox" FontSize="20" Width="120" TextChanged="ResponseTextBox_TextChanged" />
+                <TextBlock HorizontalAlignment="Center" Text="Output image size:" FontSize="12" VerticalAlignment="Center" Margin="0,0,10,0"/>
-            <Label Name="warningLabel" Visibility="Collapsed" FontStyle="Italic" Foreground="Red" HorizontalAlignment="Center">Has to be a non negative integer</Label>
+                <TextBox Margin="0,4,0,0" x:Name="ResponseTextBox_x" FontSize="12" Width="50" TextChanged="ResponseTextBox_TextChanged" />
-
+                <Label>x</Label>
-            <Button Margin="0,8,0,0" Content="OK" Click="OKButton_Click" Width="100" VerticalAlignment="Bottom"/>
+                <TextBox Margin="0,4,0,0" x:Name="ResponseTextBox_y" FontSize="12" Width="50" TextChanged="ResponseTextBox_TextChanged" IsEnabled="False" />
+            </StackPanel>
+            <Label Name="warningLabel" Visibility="Hidden"  FontStyle="Italic" Foreground="Red" HorizontalAlignment="Center" VerticalAlignment="Center">Has to be a non negative integer</Label>
+            <Button Content="OK" Click="OKButton_Click" Width="50" VerticalAlignment="Bottom" HorizontalAlignment="Right"/>
         </StackPanel>
     </Grid>
 </Window>

gui/OpenFaceOffline/UI_items/NumberEntryWindow.xaml.cs

@@ -78,20 +78,26 @@ namespace OpenFaceOffline
     /// </summary>
     public partial class NumberEntryWindow : Window
     {
-        public NumberEntryWindow()
+        public NumberEntryWindow(int initValue)
         {
             InitializeComponent();
-            ResponseTextBox.Text = "112";
+            ResponseTextBox_x.Text = initValue.ToString();
-            OutputInt = 112;
+            ResponseTextBox_y.Text = initValue.ToString();
+            OutputInt = initValue;
             this.KeyDown += new KeyEventHandler(TextEntry_KeyDown);
 
         }
 
-        private string ResponseText
+        //private string ResponseText_x
-        {
+        //{
-            get { return ResponseTextBox.Text; }
+        //    get { return ResponseTextBox_x.Text; }
-            set { ResponseTextBox.Text = value; }
+        //    set { ResponseTextBox_x.Text = value; }
-        }
+        //}
+        //private string ResponseText_y
+        //{
+        //    get { return ResponseTextBox_y.Text; }
+        //    set { ResponseTextBox_y.Text = value; }
+        //}
 
         public int OutputInt;
 
@@ -114,28 +120,29 @@ namespace OpenFaceOffline
 
             try
             {
-                OutputInt = Int32.Parse(ResponseTextBox.Text);
+                int OutputIntNew = Int32.Parse(ResponseTextBox_x.Text);
-                if(OutputInt > 0)
+                if(OutputIntNew > 0)
                 {
-                    warningLabel.Visibility = System.Windows.Visibility.Collapsed;
+                    OutputInt = OutputIntNew;
+                    warningLabel.Visibility = System.Windows.Visibility.Hidden;
                 }
                 else
                 {
                     warningLabel.Visibility = System.Windows.Visibility.Visible;
-                    OutputInt = 112;
+                    ResponseTextBox_x.Text = OutputInt.ToString();
-                    ResponseTextBox.Text = "112";
+                    ResponseTextBox_x.SelectionStart = ResponseTextBox_x.Text.Length;
-                    ResponseTextBox.SelectionStart = ResponseTextBox.Text.Length;
 
                 }
             }
             catch (FormatException except)
             {
-                OutputInt = 112;
+                ResponseTextBox_x.Text = OutputInt.ToString();
-                ResponseTextBox.Text = "112";
+                ResponseTextBox_x.SelectionStart = ResponseTextBox_x.Text.Length;
-                ResponseTextBox.SelectionStart = ResponseTextBox.Text.Length;
                 warningLabel.Visibility = System.Windows.Visibility.Visible;
             }
 
+            ResponseTextBox_y.Text = OutputInt.ToString();
+
         }
 
     }