flip image, change colours, add countdown, clickable previews

This commit is contained in:
Ruben van de Ven 2020-10-01 16:22:15 +02:00
parent a7ea09fb52
commit 8b64e8301b
4 changed files with 150 additions and 21 deletions

26
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,26 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Python: Current File",
"type": "python",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal"
},
{
"name": "Python: Mirror",
"type": "python",
"request": "launch",
"program": "mirror.py",
"args": [
// "--fullscreen",
"--camera", "2",
],
"console": "integratedTerminal"
}
]
}

31
README.md Normal file
View File

@ -0,0 +1,31 @@
A `mirror` which shows which faces are detected through three different facial detection algorithms:
* OpenCV's deep neural net [face detector](https://github.com/opencv/opencv/tree/master/samples/dnn/face_detector).
* Dlib's default frontal face detector, which is HOG based
* A Viola-Jones Haarcascade detection. Any OpenCV compatible xml file should work. It defaults to the canonical `haarcascade_frontalface_alt2.xml`.
# Installation
## on windows
The installation in Windows can be done, though it is quite elaborate:
* Install rustup-init
* Install VS C++
* Install python3
* Install Cmake (needed for python dlib)
+ make sure to add it to path
* Install git
+ including ssh deploy key
* `git clone https://git.rubenvandeven.com/r/face_detector`
* `cd face_recognition`
* `git submodules init`
* `git submodules update`
* `pip install virtualenv`
* `virtualenv.exe venv`
* `.\venv\Scripts\activate`
* `cd .\dnn\face_detector`
* `python.exe .\download_weights.py`
* `cd .\visualhaar`
* `cargo build --lib`

View File

@ -5,14 +5,15 @@ import logging
import argparse import argparse
import numpy as np import numpy as np
import time import time
import math
import datetime import datetime
from PIL import ImageFont, ImageDraw, Image from PIL import ImageFont, ImageDraw, Image
import os import os
draw_colors = { draw_colors = {
'hog': (198,65,124), 'hog': (255,255,255), #(198,65,124),
'haar': (255,255,255), 'haar': (255,255,255),
'dnn': (251,212,36), 'dnn': (255,255,255) #(251,212,36),
} }
titles = { titles = {
@ -25,6 +26,7 @@ fontfile = "SourceSansPro-Regular.ttf"
font = ImageFont.truetype(fontfile, 30) font = ImageFont.truetype(fontfile, 30)
font_s = ImageFont.truetype(fontfile, 20) font_s = ImageFont.truetype(fontfile, 20)
countdown_font = ImageFont.truetype(fontfile, 160)
class Result(): class Result():
def __init__(self, algorithm, image, confidence_threshold = 0.5): def __init__(self, algorithm, image, confidence_threshold = 0.5):
@ -80,7 +82,7 @@ class Result():
alpha = 1 alpha = 1
else: else:
# At least 10% opacity # At least 10% opacity
alpha = max(.3, detection['confidence']) alpha = max(.2, detection['confidence'])
color = list(color) color = list(color)
color.append(int(alpha*255)) color.append(int(alpha*255))
@ -119,6 +121,8 @@ def record(device_id, q1,q2, q3, q4, resolution, rotate):
capture.set(cv2.CAP_PROP_FRAME_WIDTH, resolution[1] if is_rotated_90 else resolution[0]) capture.set(cv2.CAP_PROP_FRAME_WIDTH, resolution[1] if is_rotated_90 else resolution[0])
capture.set(cv2.CAP_PROP_FRAME_HEIGHT, resolution[0] if is_rotated_90 else resolution[1]) capture.set(cv2.CAP_PROP_FRAME_HEIGHT, resolution[0] if is_rotated_90 else resolution[1])
gave_camera_warning = False
while True: while True:
ret, image = capture.read() ret, image = capture.read()
@ -130,9 +134,13 @@ def record(device_id, q1,q2, q3, q4, resolution, rotate):
if rotate is not None: if rotate is not None:
image = cv2.rotate(image, rotate) image = cv2.rotate(image, rotate)
# Flip image to create the 'mirror' effect.
image = cv2.flip(image, 1)
# print(image.shape[:2], image.shape[1::-1]) # print(image.shape[:2], image.shape[1::-1])
if image.shape[1::-1] != resolution: if image.shape[1::-1] != resolution and not gave_camera_warning:
logging.warning(f"Camera resultion seems wrong: {image.shape[:2]} instead of {resolution}") logging.warning(f"Camera resultion seems wrong: {image.shape[:2]} instead of {resolution}")
gave_camera_warning = True
try: try:
q1.put_nowait(image) q1.put_nowait(image)
@ -283,7 +291,7 @@ def process2_dnn(in_q, out_q):
prototxt = "dnn/face_detector/opencv_face_detector.pbtxt" prototxt = "dnn/face_detector/opencv_face_detector.pbtxt"
prototxt = "dnn/face_detector/deploy.prototxt" prototxt = "dnn/face_detector/deploy.prototxt"
model = "dnn/face_detector/res10_300x300_ssd_iter_140000_fp16.caffemodel" model = "dnn/face_detector/res10_300x300_ssd_iter_140000_fp16.caffemodel"
confidence_threshold = 0.5 confidence_threshold = 0.7
logger.info("[INFO] loding model...") logger.info("[INFO] loding model...")
net = cv2.dnn.readNetFromCaffe(prototxt, model) net = cv2.dnn.readNetFromCaffe(prototxt, model)
@ -371,7 +379,7 @@ def process3_haar(in_q, out_q, cascade_file):
frame = in_q.get() frame = in_q.get()
(height_orig, width_orig) = frame.shape[:2] (height_orig, width_orig) = frame.shape[:2]
scale_factor = 3 scale_factor = 4
width = int(width_orig/scale_factor) width = int(width_orig/scale_factor)
height = int(height_orig/scale_factor) height = int(height_orig/scale_factor)
@ -426,7 +434,7 @@ def process3_haar(in_q, out_q, cascade_file):
# print(img) # print(img)
out_q.put(result) out_q.put(result)
def draw_stats(image, results): def draw_stats(image, results, padding):
pil_im = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) pil_im = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
draw = ImageDraw.Draw(pil_im, 'RGBA') draw = ImageDraw.Draw(pil_im, 'RGBA')
@ -437,7 +445,8 @@ def draw_stats(image, results):
c = result.count_detections() c = result.count_detections()
txt = "face" if c == 1 else "faces" txt = "face" if c == 1 else "faces"
txt = f"{result.algorithm.ljust(5)} {c} {txt}" txt = f"{result.algorithm.ljust(5)} {c} {txt}"
draw.text((10, pil_im.size[1] - i*25 - 50), txt, fill=draw_colors[result.algorithm], font=font_s, stroke_width=1, stroke_fill=(0,0,0)) height = padding + 25
draw.text((padding, pil_im.size[1] - i*height - height), txt, fill=draw_colors[result.algorithm], font=font_s, stroke_width=1, stroke_fill=(0,0,0))
return cv2.cvtColor(np.array(pil_im), cv2.COLOR_RGB2BGR) return cv2.cvtColor(np.array(pil_im), cv2.COLOR_RGB2BGR)
@ -447,19 +456,50 @@ def display(image_res, q1, q2, q3, q4, fullscreen, output_dir):
empty_image = np.zeros((image_res[1],image_res[0],3), np.uint8) empty_image = np.zeros((image_res[1],image_res[0],3), np.uint8)
image_ratio = image_res[0] / image_res[1]
results = [None, None, None] results = [None, None, None]
result_queues = [q2, q3, q4] result_queues = [q2, q3, q4]
images = [empty_image, empty_image, empty_image, empty_image] images = [empty_image, empty_image, empty_image, empty_image]
override_image = None override_image = None
override_until = None override_until = None
countdown_until = None
# imageIdx = 0
# grid in the right corner
preview_scale = 10
preview_width = round(image_res[0] / preview_scale)
preview_height = round(preview_width / image_ratio)
padding = round(image_res[0] / 100)
if fullscreen: if fullscreen:
cv2.namedWindow("output", cv2.WND_PROP_FULLSCREEN) cv2.namedWindow("output", cv2.WINDOW_NORMAL)
cv2.setWindowProperty("output",cv2.WND_PROP_FULLSCREEN,cv2.WINDOW_FULLSCREEN) cv2.setWindowProperty("output",cv2.WND_PROP_FULLSCREEN,cv2.WINDOW_FULLSCREEN)
else:
cv2.namedWindow("output", cv2.WINDOW_AUTOSIZE)
def selectPreview(event, x, y, flags, param):
if event == cv2.EVENT_LBUTTONDOWN:
if x > image_res[0] - padding or x < image_res[0] - padding - preview_width:
return
preview_images = [idx for idx,image in enumerate(images) if idx != selectPreview.imageIdx]
for offset, image_nr in enumerate(preview_images):
offset_y = (preview_height + padding) * offset
# print(offset, y, image_res[0] - padding - preview_height - offset_y, image_res[0] - padding - offset_y)
if y > image_res[1] - padding - preview_height - offset_y and y < image_res[1] - padding - offset_y:
selectPreview.imageIdx = image_nr
print("Select image", offset, image_nr)
break
selectPreview.imageIdx = 0
cv2.setMouseCallback('output', selectPreview)
while True: while True:
logging.debug('r')
try: try:
image = q1.get_nowait() image = q1.get_nowait()
images[0] = cv2.resize(image, (image_res[0], image_res[1])) images[0] = cv2.resize(image, (image_res[0], image_res[1]))
@ -481,30 +521,60 @@ def display(image_res, q1, q2, q3, q4, fullscreen, output_dir):
else: else:
override_image = None override_image = None
images[0] = draw_stats(images[0], results) # images[0] = draw_stats(images[0], results)
img_concate_Verti1 = np.concatenate((images[0],images[1]),axis=0) # show the selected image:
img_concate_Verti2 = np.concatenate((images[2],images[3]),axis=0) grid_img = images[selectPreview.imageIdx].copy()
grid_img = np.concatenate((img_concate_Verti1,img_concate_Verti2),axis=1)
# previews in the right bottom corner
preview_images = [image for idx,image in enumerate(images) if idx != selectPreview.imageIdx]
for idx, image in enumerate(preview_images):
offset_y = (preview_height + padding) * idx
grid_img[
grid_img.shape[0] - padding - preview_height - offset_y:grid_img.shape[0] - padding - offset_y,
grid_img.shape[1] - padding - preview_width:grid_img.shape[1] - padding] = cv2.resize(image, (preview_width, preview_height), cv2.INTER_CUBIC)
# statistics
grid_img = draw_stats(grid_img, results, padding)
pil_im = Image.fromarray(cv2.cvtColor(grid_img, cv2.COLOR_BGR2RGB))
draw = ImageDraw.Draw(pil_im, 'RGBA')
# Draw countdown
if countdown_until:
duration = math.ceil(countdown_until - time.time())
w, h = draw.textsize(f"{duration}", font=countdown_font)
draw.text(((grid_img.shape[1]-w)/2,(grid_img.shape[0]-h)/2), f"{duration}", fill="white", stroke="black", font=countdown_font)
grid_img = cv2.cvtColor(np.array(pil_im), cv2.COLOR_RGB2BGR)
# img_concate_Verti1 = np.concatenate((images[0],images[1]),axis=0)
# img_concate_Verti2 = np.concatenate((images[2],images[3]),axis=0)
# grid_img = np.concatenate((img_concate_Verti1,img_concate_Verti2),axis=1)
cv2.imshow("output", grid_img) cv2.imshow("output", grid_img)
# Hit 'q' on the keyboard to quit! # Hit 'q' on the keyboard to quit!
key = cv2.waitKey(1) & 0xFF key = cv2.waitKey(1) & 0xFF
if key == ord('q'): if key == ord('q'):
break break
if key == ord(' '): if key == ord(' ') and not override_image:
countdown_until = time.time() + 3 # seconds of countdown
if countdown_until is not None and time.time() > countdown_until:
countdown_until = None
# TODO wait for frame to be processed. Eg. if I move and make a pic, it should use the last frame... # TODO wait for frame to be processed. Eg. if I move and make a pic, it should use the last frame...
output_res = (image_res[0] *2, image_res[1] * 2) # SNAP!
# output_res = (image_res[0] *2, image_res[1] * 2)
output_res = image_res # no scaling needed anyore
pil_im = Image.fromarray(cv2.cvtColor(images[0], cv2.COLOR_BGR2RGB)) pil_im = Image.fromarray(cv2.cvtColor(images[0], cv2.COLOR_BGR2RGB))
pil_im = pil_im.resize(output_res) pil_im = pil_im.resize(output_res)
draw = ImageDraw.Draw(pil_im, 'RGBA') draw = ImageDraw.Draw(pil_im, 'RGBA')
for result in results: for result in results:
if result is None: if result is None:
continue continue
result.resize(output_res[0], output_res[1]).draw_detections_on(draw) result.resize(output_res[0], output_res[1]).draw_detections_on(draw)
override_image = cv2.cvtColor(np.array(pil_im), cv2.COLOR_RGB2BGR) override_image = cv2.cvtColor(np.array(pil_im), cv2.COLOR_RGB2BGR)
override_until = time.time() + 5 override_until = time.time() + 5
logger.info("Show frame until %f", override_until) logger.info("Show frame until %f", override_until)
@ -514,10 +584,12 @@ def display(image_res, q1, q2, q3, q4, fullscreen, output_dir):
cv2.imwrite(os.path.join(output_dir, f'{name}.png'),override_image) cv2.imwrite(os.path.join(output_dir, f'{name}.png'),override_image)
for result in results: for result in results:
cv2.imwrite(os.path.join(output_dir, f'{name}-{result.algorithm}.png'),result.visualisation) cv2.imwrite(os.path.join(output_dir, f'{name}-{result.algorithm}.png'),result.visualisation)
def main(camera_id, rotate, fullscreen, cascade_file, output_dir): def main(camera_id, rotate, fullscreen, cascade_file, output_dir):
image_size = (int(1920/2), int(1080/2)) image_size = (1920, 1080) #(int(1920/2), int(1080/2))
if not os.path.exists(cascade_file): if not os.path.exists(cascade_file):
raise RuntimeError(f"Cannot load OpenCV haar-cascade file '{cascade_file}'") raise RuntimeError(f"Cannot load OpenCV haar-cascade file '{cascade_file}'")

@ -1 +1 @@
Subproject commit 928da82d24de1ae2cef268c140f9992b0614806b Subproject commit 7c7ae29bf9e1390ea304e3708e8f92f6d57f87ff