moodmeter/parse_output.py

581 lines
17 KiB
Python
Raw Normal View History

2017-11-17 16:17:31 +01:00
import os
from PIL import Image, ImageDraw,ImageTk
2017-11-17 16:17:31 +01:00
import argparse
import json
2017-11-20 16:39:28 +01:00
import time
import glob
import numpy as np
2017-12-04 21:47:24 +01:00
import datetime
import Tkinter
2017-12-04 21:47:24 +01:00
import moviepy.editor as mpy
import sys
import wave
FPS = 1
facialParameters = [
"smile",
"innerBrowRaise",
"browRaise",
"browFurrow",
"noseWrinkle",
"upperLipRaise",
"lipCornerDepressor",
"chinRaise",
"lipPucker",
"lipPress",
"lipSuck",
"mouthOpen",
"smirk",
#~ "attention",
"eyeClosure",
"eyeWiden",
"cheekRaise",
"lidTighten",
"dimpler",
"lipStretch",
"jawDrop",
]
2017-11-17 16:17:31 +01:00
2017-12-04 21:47:24 +01:00
defaultFacialParameters = [
"smile",
"innerBrowRaise",
"browRaise",
"browFurrow",
"noseWrinkle",
"upperLipRaise",
"lipCornerDepressor",
"chinRaise",
#~ "lipPucker",
"lipPress",
"lipSuck",
"mouthOpen",
#~ "smirk",
#~ "attention",
"eyeClosure",
"eyeWiden",
"cheekRaise", #
"lidTighten",
"dimpler", #
"lipStretch",
"jawDrop", #
]
2017-11-17 16:17:31 +01:00
parser = argparse.ArgumentParser(description='Parses opencv-webcam-demo json output files and collects statistics')
parser.add_argument('--frameOutput', '-o', required=True, help='directory to look for frames & json')
2017-12-04 21:47:24 +01:00
parser.add_argument('--targetDir', help='Directory in which files are stored')
parser.add_argument('--status', action='store_true', help='Keep status of last frame')
2017-12-04 21:47:24 +01:00
parser.add_argument('--stats', action='store_true', help='Show statistics for all frames')
2017-11-20 16:39:28 +01:00
parser.add_argument('--cutAllFaces', action='store_true', help='Cut out all faces from all frames')
parser.add_argument('--sum', action='store_true', help='Get total scores over all time')
parser.add_argument('--unique', action='store_true', help='Get most unique window')
parser.add_argument('--avg', action='store_true', help='Get most average window')
2017-12-04 21:47:24 +01:00
parser.add_argument('--json', action='store_true', help='Get output as JSON for graphs (graphs.php)')
parser.add_argument('--wav', '-w', required=False, default=False, help='split wave file into windows')
2017-11-20 16:39:28 +01:00
parser.add_argument('--disonant', action='store_true', help='Get most disonant faces over time')
parser.add_argument('--window-size', '-s', type=int, default=10, help='The nr of frames to group in one sliding window for analysis')
2017-12-04 21:47:24 +01:00
parser.add_argument("--params", "--metrics", "-p", type=str, nargs='+', default=defaultFacialParameters, choices=facialParameters, help="The parameters used to calculate the statistics")
2017-11-17 16:17:31 +01:00
args = parser.parse_args()
2017-11-17 16:41:51 +01:00
faces = []
2017-11-17 16:17:31 +01:00
class Face:
def __init__(self, frame, data):
self.id = data['id']
self.frame = frame # Frame class
self.data = data # json data
2017-11-20 16:39:28 +01:00
self.disonanceScore = None # a first attempt, can be deprecated?
self.anomalyScore = None
2017-11-17 16:17:31 +01:00
def getFaceImg(self):
r = self.data['rect']
return self.frame.getImg().crop((int(r['x']), int(r['y']), int(r['x']+r['w']), int(r['y']+r['h'])))
2017-11-20 16:39:28 +01:00
def getCharacteristicVector(self, params):
self.vector = [self.data[p] for p in params]
2017-11-20 16:39:28 +01:00
return self.vector
def setAnomalyScore(self, score):
self.anomalyScore = score
2017-11-17 16:17:31 +01:00
class Frame:
"""
Everything for an analysed frame
"""
def __init__(self, outputPath, nr):
self.outputPath = outputPath
self.nr = nr
self.name = "frame%06d" % nr
self.jsonPath = os.path.join(outputPath, ("frame%06d" % (nr)) + ".json")
self.imgPath = os.path.join(outputPath, self.name + ".jpg")
self.faces = None # init with getFaces
def getTime(self):
2017-12-04 21:47:24 +01:00
"""
@return datetime
"""
with Image.open(self.imgPath) as i:
#~ print(i._getexif()[36867])
return datetime.datetime.strptime(i._getexif()[36867], "%Y:%m:%d %H:%M:%S")
#~ return datetime.datetime.fromtimestamp(os.path.getmtime(self.imgPath))
2017-11-17 16:17:31 +01:00
def getJson(self):
#~ try:
with open(self.jsonPath) as fp:
return json.load(fp)
#~ except Exception as e:
#~ # no json file yet?
#~ return None
2017-12-04 21:47:24 +01:00
def getImg(self, markFaces = False):
img = Image.open(self.imgPath)
if not markFaces:
return img
draw = ImageDraw.Draw(img)
for f in self.faces:
xy1 = (int(f.data['rect']['x']), int(f.data['rect']['y']))
xy2 = (int(f.data['rect']['x'] + f.data['rect']['w']), int(f.data['rect']['y'] + f.data['rect']['h']))
draw.rectangle([xy1, xy2], outline="#ff0000")
return img
2017-11-17 16:17:31 +01:00
def getFaces(self):
if self.faces is None:
j = self.getJson()
self.faces = [Face(self, f) for f in j['faces']]
2017-11-17 16:41:51 +01:00
faces.extend(self.faces)
2017-11-17 16:17:31 +01:00
return self.faces
2017-11-17 16:41:51 +01:00
def updateDisonanceScores(self):
totalValence = 0.0
totalFaces = 0
for face in self.getFaces():
totalValence += face.data['valence']
totalFaces += 1
if totalFaces == 0:
return
avgValence = totalValence / totalFaces
for face in self.getFaces():
face.disonanceScore = abs(face.data['valence'] - avgValence)
2017-11-20 16:39:28 +01:00
def getAverageV(self, params):
vectors = [face.getCharacteristicVector(params) for face in self.getFaces()]
2017-11-20 16:39:28 +01:00
vAvg = np.mean(vectors, axis=0)
return vAvg
2017-12-04 21:47:24 +01:00
def getStdDev(self, params):
"""
Get standard deviation of the faces within the frame for given params
"""
vectors = [f.getCharacteristicVector(params) for f in self.getFaces()]
return np.std(vectors)
def updateAnomalyScores(self, params):
vAvg = self.getAverageV(params)
2017-11-20 16:39:28 +01:00
for face in self.getFaces():
face.setAnomalyScore(np.linalg.norm(face.getCharacteristicVector() - vAvg))
2017-11-17 16:41:51 +01:00
2017-11-17 16:17:31 +01:00
def exists(self):
return os.path.exists(self.jsonPath) and os.path.exists(self.imgPath)
frames = {}
class Window:
def __init__(self, frameSubset):
"""
Init a sliding window for given Frame-s
"""
self.frames = frameSubset
self.deviation = None
self.standardDeviation = None
2017-12-04 21:47:24 +01:00
def getStartTime(self):
"""get time of first frame in window. Returns datetime"""
return self.frames[0].getTime()
def getEndTime(self):
"""get time of last frame in window. Returns datetime"""
return self.frames[-1].getTime()
def getFaces(self):
faces = []
for frame in self.frames:
faces.extend(frame.getFaces())
return faces
def getStdDev(self, params):
"""
Get standard deviation of the faces within the window for given params
"""
vectors = [f.getCharacteristicVector(params) for f in self.getFaces()]
return np.std(vectors)
def getAverageV(self, params):
vectors = [f.getCharacteristicVector(params) for f in self.getFaces()]
vAvg = np.mean(vectors, axis=0)
return vAvg
@staticmethod
def createWindows(windowSize, frames):
"""
Give a full list of frames and tunrn it into a collection of sliding windows
"""
frames = sorted(frames.items(), key=lambda f: f[0])
frames = [f[1] for f in frames]
windows = []
2017-12-04 21:47:24 +01:00
windowCount = len(frames) / windowSize
#~ windowCount = len(frames) - windowSize + 1
if windowCount < 1:
raise Exception("Not enough frames ({}) for a window of size {}".format(len(frames), windowSize))
for offset in range(0, windowCount):
2017-12-04 21:47:24 +01:00
frameSubset = frames[offset*windowSize:(offset+1)*windowSize]
#~ frameSubset = frames[offset:offset+windowSize]
windows.append(Window(frameSubset))
return windows
class WindowCollection:
def __init__(self, windowSize, frames):
self.windows = Window.createWindows(windowSize, frames)
self.frames = frames
#~ self.faces = [face for face in frame.getFaces() for frame in frames]
#~ def getMostWindowsClosestToMedian(self, nr = 5):
#~ """
#~ Get windows with the faces closest to the median
#~ """
#~ self.faces
def getWindowVectors(self, params):
return [window.getAverageV(params) for window in self.windows]
def getWindowsByDeviation(self, params):
vectors = self.getWindowVectors(params)
vAvg = np.mean(vectors, axis=0)
#~ diffs = [numpy.linalg.norm(v-vAvg) for v in vectors]
#~ min_index, min_value = min(enumerate(diffs), key=lambda p: p[1])
#~ max_index, max_value = max(enumerate(diffs), key=lambda p: p[1])
return sorted(self.windows, key=lambda w: np.linalg.norm(w.getAverageV(params)-vAvg))
2017-12-04 21:47:24 +01:00
def getDeviationForWindow(self, window, params):
vectors = self.getWindowVectors(params)
vAvg = np.mean(vectors, axis=0)
return np.linalg.norm(w.getAverageV(params)-vAvg)
def getUniqueWindows(self, params, nr=5):
windows = self.getWindowsByDeviation(params)
return windows[0: nr]
def getMostAvgWindows(self, params, nr=5):
windows = self.getWindowsByDeviation(params)
windows.reverse()
return windows[0:nr]
def getMostContrastingWindows(self, params, nr=5):
sortedWindows = sorted(self.windows, key=lambda w: w.getStdDev(params), reverse=True)
return sortedWindows[0:nr]
2017-11-17 16:17:31 +01:00
def loadFrames(frameDir):
global frames
nr = 2
nextFrame = Frame(frameDir, nr)
# TODO; make threaded and infinite loop that updates global frames
while nextFrame.exists():
frames[nr] = nextFrame
nr+=1
nextFrame = Frame(frameDir, nr)
return frames
2017-11-20 16:39:28 +01:00
def getLastFrame(frameDir):
jsons = sorted(glob.glob(os.path.join(frameDir, "*.json")))
if len(jsons):
lastJson = jsons[-1]
lastNr = int(lastJson[-11:-5])
frame = Frame(frameDir, lastNr)
return frame
return None
2017-11-17 16:17:31 +01:00
def cutOutFaces(frame, targetDir):
for faceNr, face in enumerate(frame.getFaces()):
print(faceNr, face)
img = face.getFaceImg()
faceImgPath = os.path.join(targetDir, frame.name + "-%s.jpg" % face.id)
print(faceImgPath)
img.save(faceImgPath)
pass
2017-11-20 16:39:28 +01:00
def validateJsonTimes():
lastTime = None
for frameNr, frame in loadFrames(args.frameOutput).items():
thisTime = frame.getJson()['t']
#print(frameNr, thisTime)
if not (lastTime is None) and lastTime > thisTime:
2017-12-04 21:47:24 +01:00
sys.stderr.write("ERRROR!! Time error at %s. Restarted scanner there?\n" % frameNr)
2017-11-20 16:39:28 +01:00
lastTime = thisTime
2017-11-17 16:17:31 +01:00
def sumEmotions():
total = 0.
summed = 0.
items = 0
2017-11-20 16:39:28 +01:00
for frameNr, frame in loadFrames(args.frameOutput).items():
2017-11-17 16:17:31 +01:00
for face in frame.getFaces():
total += abs(face.data['valence'])
summed += face.data['valence']
items += 1
average = summed / items
print ("Total emotion %d, positivity score %d (average: %s)" % (total, summed, average))
2017-11-17 16:41:51 +01:00
def getMostDisonant(nr = 5):
2017-11-20 16:39:28 +01:00
for frameNr, frame in loadFrames(args.frameOutput).items():
2017-11-17 16:41:51 +01:00
frame.updateDisonanceScores()
faces.sort(key=lambda x: x.disonanceScore, reverse=True)
2017-11-20 16:39:28 +01:00
mostDisonantFaces = faces[:nr]
2017-11-17 16:41:51 +01:00
for face in mostDisonantFaces:
print("Frame %d, face %d, score %d, valence %d" % (face.frame.nr, face.id, face.disonanceScore, face.data['valence']))
face.getFaceImg().show()
2017-11-20 16:39:28 +01:00
def getAnomalies(params, nr = 5):
2017-11-20 16:39:28 +01:00
for frameNr, frame in loadFrames(args.frameOutput).items():
frame.updateAnomalyScores(params)
2017-11-20 16:39:28 +01:00
faces.sort(key=lambda x: x.anomalyScore, reverse=True)
anomalies = faces[:nr]
for face in anomalies:
print("Frame %d, face %d, score %d" % (face.frame.nr, face.id, face.anomalyScore))
#~ getCharacteristicVector
face.getFaceImg().show()
def printFrameStats(frame, params):
2017-11-20 16:39:28 +01:00
os.system('clear')
print(time.time())
print( ("Nr: %d" % frame.nr).ljust(40) + ("t: {}".format(frame.getJson()['t'])) )
2017-11-20 16:39:28 +01:00
#~ print
faces = frame.getFaces()
print("Faces: %d" % len(faces))
if len(faces) < 1:
return
2017-11-17 16:41:51 +01:00
print " ".ljust(20), "0%".rjust(13), "q1".rjust(13), "median".rjust(13), "q3".rjust(13), "100%".rjust(13)
2017-11-20 16:39:28 +01:00
for p in params:
q0 = np.percentile(np.array([f.data[p] for f in faces]),0)
q1 = np.percentile(np.array([f.data[p] for f in faces]),25)
q2 = np.percentile(np.array([f.data[p] for f in faces]),50)
q3 = np.percentile(np.array([f.data[p] for f in faces]),75)
q4 = np.percentile(np.array([f.data[p] for f in faces]),100)
print p.ljust(20), ("%f%%" % q0).rjust(13), ("%f%%" % q1).rjust(13),("%f%%" % q2).rjust(13),("%f%%" % q3).rjust(13),("%f%%" % q4).rjust(13)
2017-11-20 16:39:28 +01:00
#~ TODO: speaker stats
frame.updateDisonanceScores()
dissonantFace = max(faces,key=lambda f: f.disonanceScore)
#~ dissonantFace.getFaceImg()
def monitorStatus(frameDir, params):
2017-11-20 16:39:28 +01:00
while True:
frame = getLastFrame(frameDir)
if not frame is None:
printFrameStats(frame, params)
2017-11-20 16:39:28 +01:00
# don't check too often
time.sleep(.5)
2017-12-04 21:47:24 +01:00
def printStats(frameDir, params):
frames = loadFrames(frameDir)
vAvg = np.mean([f.getAverageV(params) for i,f in frames.items()], axis=0)
for nr, frame in frames.items():
max_index, max_value = max(enumerate(frame.getAverageV(params)), key=lambda a: a[1])
distance = np.linalg.norm(vAvg - frame.getAverageV(params))
print("{:06d}: {: >4} faces (d: {: 6.2f}, internal s {: 6.2f}, max: {})".format(frame.nr, len(frame.getFaces()),distance, frame.getStdDev(params), params[max_index]))
print("{} frames".format(len(frames)))
def playWindowStopmotion(window, params):
"""
Play a set of sliding window frames as stop motion video
"""
root = Tkinter.Tk()
root.geometry('%dx%d+%d+%d' % (1000,1000,0,0))
canvas = Tkinter.Canvas(root,width=1000,height=1000)
canvas.pack()
old_label_image = None
for frame in window.frames:
2017-12-04 21:47:24 +01:00
image = frame.getImg(markFaces=True)
basewidth = 1000
wpercent = (basewidth / float(image.size[0]))
hsize = int((float(image.size[1]) * float(wpercent)))
image = image.resize((basewidth, hsize), Image.ANTIALIAS)
tkpi = ImageTk.PhotoImage(image)
canvas.delete("IMG")
imagesprite = canvas.create_image(500,500,image=tkpi, tags="IMG")
root.update()
time.sleep(1)
2017-11-20 16:39:28 +01:00
2017-12-04 21:47:24 +01:00
def createWindowVideo(window, params, targetDir):
global FPS
basewidth = 1920
i=0
images = []
for frame in window.frames:
image = frame.getImg(markFaces=True)
wpercent = (basewidth / float(image.size[0]))
hsize = int((float(image.size[1]) * float(wpercent)))
image = image.resize((basewidth, hsize), Image.ANTIALIAS)
imgName = os.path.join("/tmp/","renderframe{:06d}.jpg".format(i))
print("(VIDEO) Create frame: {}".format(imgName))
image.save(imgName)
images.append(imgName)
i+=1
clip = mpy.ImageSequenceClip(images, fps=FPS)
#~ audio = mpy.AudioFileClip(filename="")
#~ audio.set_start(...)
#~ audio.set_duration(len(images)/FPS)
#~ clip = clip.set_audio(audio)
targetName = "video_" + "-".join(params) + ".mp4"
print("(VIDEO) Write video: {}".format(targetName))
clip.write_videofile(os.path.join(targetDir,targetName), fps=FPS)
def getGraphData(frames, params, window_size, outputNr):
"""
Get data to generate graph with
"""
collection = WindowCollection(window_size, frames)
windows = collection.windows
jsonOutput = {'windows':[], 'outputNr': outputNr}
for window in windows:
w = {
'start': window.getStartTime().strftime('%Y-%m-%d %H:%M:%S'),
'end': window.getEndTime().strftime('%Y-%m-%d %H:%M:%S'),
'startFrame': window.frames[0].name,
'params': {},
'param_values': {},
}
pVs = []
for param in params:
paramV = window.getAverageV([param])
value = paramV[0]
pVs.append(value)
w['params'][param] = value
#~ values = []
#~ for frame in window.frames:
#~ for face in frame.getFaces():
#~ values.append(face.data[param])
#~ w['param_values'][param] = values
w['avg'] = np.mean(pVs)
w['stddev'] = window.getStdDev(params)
jsonOutput['windows'].append(w)
jsonOutput['range'] = [
jsonOutput['windows'][0]['start'],
jsonOutput['windows'][-1]['end'],
]
return jsonOutput
def splitAudio(frames, window_size, srcWav, targetDir):
"""
Get audio cllips
"""
global FPS
if not os.path.exists(srcWav):
raise Exception("Wav does not exist")
if not os.path.exists(targetDir):
raise Exception("Target dir does not exist")
collection = WindowCollection(window_size, frames)
origAudio = wave.open(srcWav,'r')
frameRate = origAudio.getframerate()
nChannels = origAudio.getnchannels()
sampWidth = origAudio.getsampwidth()
windows = collection.windows
for i, window in enumerate(windows):
print("Window {}, {}".format(i, window.frames[0].name))
start = i * window_size * FPS
end = i+1 * window_size * FPS
origAudio.setpos(start*frameRate)
print(origAudio.tell())
chunkData = origAudio.readframes(int((end-start)*frameRate))
print(origAudio.tell())
targetFile = os.path.join(targetDir, window.frames[0].name + '.wav')
print("\t{}".format(targetFile))
chunkAudio = wave.open(targetFile,'w')
chunkAudio.setnchannels(nChannels)
chunkAudio.setsampwidth(sampWidth)
chunkAudio.setframerate(frameRate)
chunkAudio.writeframes(chunkData)
chunkAudio.close()
#~ def createGraphPage(windows, params, targetFilename):
#~ jsonOutput = {'windows':[]}
#~ for window in windows:
#~ window
2017-11-20 16:39:28 +01:00
validateJsonTimes()
if args.sum:
sumEmotions()
if args.disonant:
getMostDisonant()
if args.cutAllFaces:
faceDir = os.path.join(args.frameOutput, 'faces')
if not os.path.exists(faceDir):
os.mkdir(faceDir)
for frameNr, frame in loadFrames(args.frameOutput).items():
cutOutFaces(faceDir)
if args.unique:
collection = WindowCollection(args.window_size, frames)
windows = collection.getUniqueWindows(args.params)
#~ print(windows)
2017-12-04 21:47:24 +01:00
if args.targetDir:
createWindowVideo(windows[0], args.params, args.targetDir)
else:
playWindowStopmotion(windows[0], args.params)
if args.avg:
collection = WindowCollection(args.window_size, frames)
windows = collection.getMostAvgWindows(args.params)
#~ print(windows)
2017-12-04 21:47:24 +01:00
if args.targetDir:
createWindowVideo(windows[0], args.params, args.targetDir)
else:
playWindowStopmotion(windows[0], args.params)
2017-11-20 16:39:28 +01:00
if args.status:
monitorStatus(args.frameOutput, args.params)
2017-12-04 21:47:24 +01:00
if args.stats:
printStats(args.frameOutput, args.params)
if args.json:
nr = int(args.frameOutput[-1])
print json.dumps(getGraphData(frames, args.params, args.window_size, nr))
if args.wav:
print splitAudio(frames, args.window_size, args.wav, args.targetDir)