Some renaming to prepare for video scrubbing mode.
This commit is contained in:
Tadas Baltrusaitis
parent
41fc0bd43c
commit
e0f71fb67c
2 changed files with 267 additions and 143 deletions
|
|
@ -227,7 +227,7 @@ namespace OpenFaceOffline
|
||||||
file_no_ext = System.IO.Path.GetFileName(file_no_ext);
|
file_no_ext = System.IO.Path.GetFileName(file_no_ext);
|
||||||
|
|
||||||
// Start the actual processing and recording
|
// Start the actual processing and recording
|
||||||
VideoLoop(file_no_ext);
|
FeatureExtractionLoop(file_no_ext);
|
||||||
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
|
@ -291,7 +291,7 @@ namespace OpenFaceOffline
|
||||||
String file_no_ext = System.IO.Path.GetFileNameWithoutExtension(filename);
|
String file_no_ext = System.IO.Path.GetFileNameWithoutExtension(filename);
|
||||||
|
|
||||||
// Start the actual processing
|
// Start the actual processing
|
||||||
VideoLoop(file_no_ext);
|
FeatureExtractionLoop(file_no_ext);
|
||||||
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
|
@ -396,11 +396,9 @@ namespace OpenFaceOffline
|
||||||
|
|
||||||
|
|
||||||
// Capturing and processing the video frame by frame
|
// Capturing and processing the video frame by frame
|
||||||
private void VideoLoop(string output_file_name)
|
private void FeatureExtractionLoop(string output_file_name)
|
||||||
{
|
{
|
||||||
|
|
||||||
Thread.CurrentThread.IsBackground = true;
|
|
||||||
|
|
||||||
DateTime? startTime = CurrentTime;
|
DateTime? startTime = CurrentTime;
|
||||||
|
|
||||||
var lastFrameTime = CurrentTime;
|
var lastFrameTime = CurrentTime;
|
||||||
|
|
@ -456,150 +454,17 @@ namespace OpenFaceOffline
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool detectionSucceeding = ProcessFrame(clnf_model, clnf_params, frame, grayFrame, fx, fy, cx, cy);
|
detectionSucceeding = ProcessFrame(clnf_model, clnf_params, frame, grayFrame, fx, fy, cx, cy);
|
||||||
|
|
||||||
double scale = clnf_model.GetRigidParams()[0];
|
|
||||||
|
|
||||||
double confidence = (-clnf_model.GetConfidence()) / 2.0 + 0.5;
|
|
||||||
|
|
||||||
if (confidence < 0)
|
|
||||||
confidence = 0;
|
|
||||||
else if (confidence > 1)
|
|
||||||
confidence = 1;
|
|
||||||
|
|
||||||
List<double> pose = new List<double>();
|
|
||||||
clnf_model.GetPose(pose, fx, fy, cx, cy);
|
|
||||||
List<double> non_rigid_params = clnf_model.GetNonRigidParams();
|
|
||||||
|
|
||||||
// The face analysis step (for AUs and eye gaze)
|
// The face analysis step (for AUs and eye gaze)
|
||||||
face_analyser.AddNextFrame(frame, clnf_model, fx, fy, cx, cy, false, ShowAppearance, false); // TODO change
|
face_analyser.AddNextFrame(frame, clnf_model, fx, fy, cx, cy, false, ShowAppearance, false); // TODO change
|
||||||
|
|
||||||
List<Tuple<Point, Point>> lines = null;
|
|
||||||
List<Tuple<double, double>> landmarks = null;
|
|
||||||
List<Tuple<double, double>> eye_landmarks = null;
|
|
||||||
List<Tuple<Point, Point>> gaze_lines = null;
|
|
||||||
Tuple<double, double> gaze_angle = new Tuple<double, double>(0, 0);
|
|
||||||
|
|
||||||
if (detectionSucceeding)
|
|
||||||
{
|
|
||||||
landmarks = clnf_model.CalculateLandmarks();
|
|
||||||
eye_landmarks = clnf_model.CalculateEyeLandmarks();
|
|
||||||
lines = clnf_model.CalculateBox((float)fx, (float)fy, (float)cx, (float)cy);
|
|
||||||
gaze_lines = face_analyser.CalculateGazeLines(scale, (float)fx, (float)fy, (float)cx, (float)cy);
|
|
||||||
gaze_angle = face_analyser.GetGazeAngle();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Visualisation
|
|
||||||
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
|
|
||||||
{
|
|
||||||
if (ShowAUs)
|
|
||||||
{
|
|
||||||
var au_classes = face_analyser.GetCurrentAUsClass();
|
|
||||||
var au_regs = face_analyser.GetCurrentAUsReg();
|
|
||||||
|
|
||||||
auClassGraph.Update(au_classes);
|
|
||||||
|
|
||||||
var au_regs_scaled = new Dictionary<String, double>();
|
|
||||||
foreach (var au_reg in au_regs)
|
|
||||||
{
|
|
||||||
au_regs_scaled[au_reg.Key] = au_reg.Value / 5.0;
|
|
||||||
if (au_regs_scaled[au_reg.Key] < 0)
|
|
||||||
au_regs_scaled[au_reg.Key] = 0;
|
|
||||||
|
|
||||||
if (au_regs_scaled[au_reg.Key] > 1)
|
|
||||||
au_regs_scaled[au_reg.Key] = 1;
|
|
||||||
}
|
|
||||||
auRegGraph.Update(au_regs_scaled);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ShowGeometry)
|
|
||||||
{
|
|
||||||
int yaw = (int)(pose[4] * 180 / Math.PI + 0.5);
|
|
||||||
int roll = (int)(pose[5] * 180 / Math.PI + 0.5);
|
|
||||||
int pitch = (int)(pose[3] * 180 / Math.PI + 0.5);
|
|
||||||
|
|
||||||
YawLabel.Content = yaw + "°";
|
|
||||||
RollLabel.Content = roll + "°";
|
|
||||||
PitchLabel.Content = pitch + "°";
|
|
||||||
|
|
||||||
XPoseLabel.Content = (int)pose[0] + " mm";
|
|
||||||
YPoseLabel.Content = (int)pose[1] + " mm";
|
|
||||||
ZPoseLabel.Content = (int)pose[2] + " mm";
|
|
||||||
|
|
||||||
nonRigidGraph.Update(non_rigid_params);
|
|
||||||
|
|
||||||
// Update eye gaze
|
|
||||||
String x_angle = String.Format("{0:F0}°", gaze_angle.Item1 * (180.0 / Math.PI));
|
|
||||||
String y_angle = String.Format("{0:F0}°", gaze_angle.Item2 * (180.0 / Math.PI));
|
|
||||||
GazeXLabel.Content = x_angle;
|
|
||||||
GazeYLabel.Content = y_angle;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ShowTrackedVideo)
|
|
||||||
{
|
|
||||||
if (latest_img == null)
|
|
||||||
{
|
|
||||||
latest_img = frame.CreateWriteableBitmap();
|
|
||||||
}
|
|
||||||
|
|
||||||
frame.UpdateWriteableBitmap(latest_img);
|
|
||||||
|
|
||||||
video.Source = latest_img;
|
|
||||||
video.Confidence = confidence;
|
|
||||||
video.FPS = processing_fps.GetFPS();
|
|
||||||
video.Progress = progress;
|
|
||||||
|
|
||||||
if (!detectionSucceeding)
|
|
||||||
{
|
|
||||||
video.OverlayLines.Clear();
|
|
||||||
video.OverlayPoints.Clear();
|
|
||||||
video.OverlayEyePoints.Clear();
|
|
||||||
video.GazeLines.Clear();
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
video.OverlayLines = lines;
|
|
||||||
|
|
||||||
List<Point> landmark_points = new List<Point>();
|
|
||||||
foreach (var p in landmarks)
|
|
||||||
{
|
|
||||||
landmark_points.Add(new Point(p.Item1, p.Item2));
|
|
||||||
}
|
|
||||||
|
|
||||||
List<Point> eye_landmark_points = new List<Point>();
|
|
||||||
foreach (var p in eye_landmarks)
|
|
||||||
{
|
|
||||||
eye_landmark_points.Add(new Point(p.Item1, p.Item2));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
video.OverlayPoints = landmark_points;
|
|
||||||
video.OverlayEyePoints = eye_landmark_points;
|
|
||||||
video.GazeLines = gaze_lines;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ShowAppearance)
|
|
||||||
{
|
|
||||||
RawImage aligned_face = face_analyser.GetLatestAlignedFace();
|
|
||||||
RawImage hog_face = face_analyser.GetLatestHOGDescriptorVisualisation();
|
|
||||||
|
|
||||||
if (latest_aligned_face == null)
|
|
||||||
{
|
|
||||||
latest_aligned_face = aligned_face.CreateWriteableBitmap();
|
|
||||||
latest_HOG_descriptor = hog_face.CreateWriteableBitmap();
|
|
||||||
}
|
|
||||||
|
|
||||||
aligned_face.UpdateWriteableBitmap(latest_aligned_face);
|
|
||||||
hog_face.UpdateWriteableBitmap(latest_HOG_descriptor);
|
|
||||||
|
|
||||||
AlignedFace.Source = latest_aligned_face;
|
|
||||||
AlignedHOG.Source = latest_HOG_descriptor;
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
recorder.RecordFrame(clnf_model, face_analyser, detectionSucceeding, frame_id + 1, ((double)frame_id) / fps);
|
recorder.RecordFrame(clnf_model, face_analyser, detectionSucceeding, frame_id + 1, ((double)frame_id) / fps);
|
||||||
|
|
||||||
|
List<Tuple<double, double>> landmarks = clnf_model.CalculateLandmarks();
|
||||||
|
|
||||||
|
VisualizeFeatures(frame, landmarks, fx, fy, cx, cy, progress);
|
||||||
|
|
||||||
if (reset)
|
if (reset)
|
||||||
{
|
{
|
||||||
clnf_model.Reset();
|
clnf_model.Reset();
|
||||||
|
|
@ -635,6 +500,250 @@ namespace OpenFaceOffline
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Replaying the features frame by frame
|
||||||
|
private void FeatureVisualizationLoop(string input_feature_file, string input_video_file)
|
||||||
|
{
|
||||||
|
|
||||||
|
DateTime? startTime = CurrentTime;
|
||||||
|
|
||||||
|
var lastFrameTime = CurrentTime;
|
||||||
|
|
||||||
|
clnf_model.Reset();
|
||||||
|
face_analyser.Reset();
|
||||||
|
|
||||||
|
// TODO these need to be stored so that they could be loaded somewhere
|
||||||
|
double fx = 500.0 * (capture.width / 640.0);
|
||||||
|
double fy = 500.0 * (capture.height / 480.0);
|
||||||
|
|
||||||
|
fx = (fx + fy) / 2.0;
|
||||||
|
fy = fx;
|
||||||
|
|
||||||
|
double cx = capture.width / 2f;
|
||||||
|
double cy = capture.height / 2f;
|
||||||
|
|
||||||
|
int frame_id = 0;
|
||||||
|
|
||||||
|
double fps = capture.GetFPS();
|
||||||
|
if (fps <= 0) fps = 30;
|
||||||
|
|
||||||
|
while (thread_running)
|
||||||
|
{
|
||||||
|
//////////////////////////////////////////////
|
||||||
|
// CAPTURE FRAME AND DETECT LANDMARKS FOLLOWED BY THE REQUIRED IMAGE PROCESSING
|
||||||
|
//////////////////////////////////////////////
|
||||||
|
RawImage frame = null;
|
||||||
|
double progress = -1;
|
||||||
|
|
||||||
|
frame = new RawImage(capture.GetNextFrame(false));
|
||||||
|
progress = capture.GetProgress();
|
||||||
|
|
||||||
|
if (frame.Width == 0)
|
||||||
|
{
|
||||||
|
// This indicates that we reached the end of the video file
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO stop button should actually clear the video
|
||||||
|
lastFrameTime = CurrentTime;
|
||||||
|
processing_fps.AddFrame();
|
||||||
|
|
||||||
|
var grayFrame = new RawImage(capture.GetCurrentFrameGray());
|
||||||
|
|
||||||
|
if (grayFrame == null)
|
||||||
|
{
|
||||||
|
Console.WriteLine("Gray is empty");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
detectionSucceeding = ProcessFrame(clnf_model, clnf_params, frame, grayFrame, fx, fy, cx, cy);
|
||||||
|
|
||||||
|
// The face analysis step (for AUs and eye gaze)
|
||||||
|
face_analyser.AddNextFrame(frame, clnf_model, fx, fy, cx, cy, false, ShowAppearance, false); // TODO change
|
||||||
|
|
||||||
|
recorder.RecordFrame(clnf_model, face_analyser, detectionSucceeding, frame_id + 1, ((double)frame_id) / fps);
|
||||||
|
|
||||||
|
List<Tuple<double, double>> landmarks = clnf_model.CalculateLandmarks();
|
||||||
|
|
||||||
|
VisualizeFeatures(frame, landmarks, fx, fy, cx, cy, progress);
|
||||||
|
|
||||||
|
if (reset)
|
||||||
|
{
|
||||||
|
clnf_model.Reset();
|
||||||
|
face_analyser.Reset();
|
||||||
|
reset = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
while (thread_running & thread_paused && skip_frames == 0)
|
||||||
|
{
|
||||||
|
Thread.Sleep(10);
|
||||||
|
}
|
||||||
|
|
||||||
|
frame_id++;
|
||||||
|
|
||||||
|
if (skip_frames > 0)
|
||||||
|
skip_frames--;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
latest_img = null;
|
||||||
|
skip_frames = 0;
|
||||||
|
|
||||||
|
// Unpause if it's paused
|
||||||
|
if (thread_paused)
|
||||||
|
{
|
||||||
|
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
|
||||||
|
{
|
||||||
|
PauseButton_Click(null, null);
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
recorder.FinishRecording(clnf_model, face_analyser);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void VisualizeFeatures(RawImage frame, List<Tuple<double, double>> landmarks, double fx, double fy, double cx, double cy, double progress)
|
||||||
|
{
|
||||||
|
List<Tuple<Point, Point>> lines = null;
|
||||||
|
List<Tuple<double, double>> eye_landmarks = null;
|
||||||
|
List<Tuple<Point, Point>> gaze_lines = null;
|
||||||
|
Tuple<double, double> gaze_angle = new Tuple<double, double>(0, 0);
|
||||||
|
|
||||||
|
List<double> pose = new List<double>();
|
||||||
|
clnf_model.GetPose(pose, fx, fy, cx, cy);
|
||||||
|
List<double> non_rigid_params = clnf_model.GetNonRigidParams();
|
||||||
|
|
||||||
|
double confidence = (-clnf_model.GetConfidence()) / 2.0 + 0.5;
|
||||||
|
|
||||||
|
if (confidence < 0)
|
||||||
|
confidence = 0;
|
||||||
|
else if (confidence > 1)
|
||||||
|
confidence = 1;
|
||||||
|
|
||||||
|
if (detectionSucceeding)
|
||||||
|
{
|
||||||
|
|
||||||
|
eye_landmarks = clnf_model.CalculateEyeLandmarks();
|
||||||
|
lines = clnf_model.CalculateBox((float)fx, (float)fy, (float)cx, (float)cy);
|
||||||
|
|
||||||
|
double scale = clnf_model.GetRigidParams()[0];
|
||||||
|
|
||||||
|
gaze_lines = face_analyser.CalculateGazeLines(scale, (float)fx, (float)fy, (float)cx, (float)cy);
|
||||||
|
gaze_angle = face_analyser.GetGazeAngle();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Visualisation (as a separate function)
|
||||||
|
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
|
||||||
|
{
|
||||||
|
if (ShowAUs)
|
||||||
|
{
|
||||||
|
var au_classes = face_analyser.GetCurrentAUsClass();
|
||||||
|
var au_regs = face_analyser.GetCurrentAUsReg();
|
||||||
|
|
||||||
|
auClassGraph.Update(au_classes);
|
||||||
|
|
||||||
|
var au_regs_scaled = new Dictionary<String, double>();
|
||||||
|
foreach (var au_reg in au_regs)
|
||||||
|
{
|
||||||
|
au_regs_scaled[au_reg.Key] = au_reg.Value / 5.0;
|
||||||
|
if (au_regs_scaled[au_reg.Key] < 0)
|
||||||
|
au_regs_scaled[au_reg.Key] = 0;
|
||||||
|
|
||||||
|
if (au_regs_scaled[au_reg.Key] > 1)
|
||||||
|
au_regs_scaled[au_reg.Key] = 1;
|
||||||
|
}
|
||||||
|
auRegGraph.Update(au_regs_scaled);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ShowGeometry)
|
||||||
|
{
|
||||||
|
int yaw = (int)(pose[4] * 180 / Math.PI + 0.5);
|
||||||
|
int roll = (int)(pose[5] * 180 / Math.PI + 0.5);
|
||||||
|
int pitch = (int)(pose[3] * 180 / Math.PI + 0.5);
|
||||||
|
|
||||||
|
YawLabel.Content = yaw + "°";
|
||||||
|
RollLabel.Content = roll + "°";
|
||||||
|
PitchLabel.Content = pitch + "°";
|
||||||
|
|
||||||
|
XPoseLabel.Content = (int)pose[0] + " mm";
|
||||||
|
YPoseLabel.Content = (int)pose[1] + " mm";
|
||||||
|
ZPoseLabel.Content = (int)pose[2] + " mm";
|
||||||
|
|
||||||
|
nonRigidGraph.Update(non_rigid_params);
|
||||||
|
|
||||||
|
// Update eye gaze
|
||||||
|
String x_angle = String.Format("{0:F0}°", gaze_angle.Item1 * (180.0 / Math.PI));
|
||||||
|
String y_angle = String.Format("{0:F0}°", gaze_angle.Item2 * (180.0 / Math.PI));
|
||||||
|
GazeXLabel.Content = x_angle;
|
||||||
|
GazeYLabel.Content = y_angle;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ShowTrackedVideo)
|
||||||
|
{
|
||||||
|
if (latest_img == null)
|
||||||
|
{
|
||||||
|
latest_img = frame.CreateWriteableBitmap();
|
||||||
|
}
|
||||||
|
|
||||||
|
frame.UpdateWriteableBitmap(latest_img);
|
||||||
|
|
||||||
|
video.Source = latest_img;
|
||||||
|
video.Confidence = confidence;
|
||||||
|
video.FPS = processing_fps.GetFPS();
|
||||||
|
video.Progress = progress;
|
||||||
|
|
||||||
|
if (!detectionSucceeding)
|
||||||
|
{
|
||||||
|
video.OverlayLines.Clear();
|
||||||
|
video.OverlayPoints.Clear();
|
||||||
|
video.OverlayEyePoints.Clear();
|
||||||
|
video.GazeLines.Clear();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
video.OverlayLines = lines;
|
||||||
|
|
||||||
|
List<Point> landmark_points = new List<Point>();
|
||||||
|
foreach (var p in landmarks)
|
||||||
|
{
|
||||||
|
landmark_points.Add(new Point(p.Item1, p.Item2));
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Point> eye_landmark_points = new List<Point>();
|
||||||
|
foreach (var p in eye_landmarks)
|
||||||
|
{
|
||||||
|
eye_landmark_points.Add(new Point(p.Item1, p.Item2));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
video.OverlayPoints = landmark_points;
|
||||||
|
video.OverlayEyePoints = eye_landmark_points;
|
||||||
|
video.GazeLines = gaze_lines;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ShowAppearance)
|
||||||
|
{
|
||||||
|
RawImage aligned_face = face_analyser.GetLatestAlignedFace();
|
||||||
|
RawImage hog_face = face_analyser.GetLatestHOGDescriptorVisualisation();
|
||||||
|
|
||||||
|
if (latest_aligned_face == null)
|
||||||
|
{
|
||||||
|
latest_aligned_face = aligned_face.CreateWriteableBitmap();
|
||||||
|
latest_HOG_descriptor = hog_face.CreateWriteableBitmap();
|
||||||
|
}
|
||||||
|
|
||||||
|
aligned_face.UpdateWriteableBitmap(latest_aligned_face);
|
||||||
|
hog_face.UpdateWriteableBitmap(latest_HOG_descriptor);
|
||||||
|
|
||||||
|
AlignedFace.Source = latest_aligned_face;
|
||||||
|
AlignedHOG.Source = latest_HOG_descriptor;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
private void StopTracking()
|
private void StopTracking()
|
||||||
{
|
{
|
||||||
// First complete the running of the thread
|
// First complete the running of the thread
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,21 @@ namespace OpenFaceOffline
|
||||||
{
|
{
|
||||||
System.IO.Directory.CreateDirectory(root);
|
System.IO.Directory.CreateDirectory(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Write out the OF file which tells where all the relevant data is
|
||||||
|
StreamWriter out_of_file = new StreamWriter(root + "/" + filename + ".of");
|
||||||
|
|
||||||
|
//out_of_file.WriteLine("Video_file:" + )
|
||||||
|
out_of_file.WriteLine("CSV file: " + root + "/" + filename + ".csv");
|
||||||
|
if(record_HOG)
|
||||||
|
{
|
||||||
|
out_of_file.WriteLine("HOG file: " + root + "/" + filename + ".hog");
|
||||||
|
}
|
||||||
|
if(record_aligned)
|
||||||
|
{
|
||||||
|
out_of_file.WriteLine("Aligned dir: " + root + "/" + filename + "/");
|
||||||
|
}
|
||||||
|
|
||||||
out_filename = root + "/" + filename + ".csv";
|
out_filename = root + "/" + filename + ".csv";
|
||||||
output_features_file = new StreamWriter(out_filename);
|
output_features_file = new StreamWriter(out_filename);
|
||||||
output_features_file.Write("frame, timestamp, confidence, success");
|
output_features_file.Write("frame, timestamp, confidence, success");
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue