566 lines
No EOL
25 KiB
Python
566 lines
No EOL
25 KiB
Python
# used for "Forward Referencing of type annotations"
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
import ffmpeg
|
|
from argparse import Namespace
|
|
import datetime
|
|
import logging
|
|
from multiprocessing import Event
|
|
from multiprocessing.synchronize import Event as BaseEvent
|
|
import cv2
|
|
import numpy as np
|
|
import json
|
|
import pyglet
|
|
import pyglet.event
|
|
import zmq
|
|
import tempfile
|
|
from pathlib import Path
|
|
import shutil
|
|
import math
|
|
from typing import Dict, Iterable, Optional
|
|
|
|
|
|
from pyglet import shapes
|
|
from PIL import Image
|
|
|
|
from trap.counter import CounterListerner
|
|
from trap.frame_emitter import DetectionState, Frame, Track, Camera
|
|
from trap.preview_renderer import FrameWriter
|
|
from trap.tools import draw_track, draw_track_predictions, draw_track_projected, draw_trackjectron_history, to_point
|
|
from trap.utils import convert_world_points_to_img_points, convert_world_space_to_img_space
|
|
|
|
|
|
|
|
logger = logging.getLogger("trap.simple_renderer")
|
|
|
|
class CvRenderer:
|
|
def __init__(self, config: Namespace, is_running: BaseEvent):
|
|
self.config = config
|
|
self.is_running = is_running
|
|
|
|
self.counter_listener = CounterListerner()
|
|
|
|
context = zmq.Context()
|
|
self.prediction_sock = context.socket(zmq.SUB)
|
|
self.prediction_sock.setsockopt(zmq.CONFLATE, 1) # only keep latest frame. NB. make sure this comes BEFORE connect, otherwise it's ignored!!
|
|
self.prediction_sock.setsockopt(zmq.SUBSCRIBE, b'')
|
|
# self.prediction_sock.connect(config.zmq_prediction_addr if not self.config.bypass_prediction else config.zmq_trajectory_addr)
|
|
self.prediction_sock.connect(config.zmq_prediction_addr)
|
|
|
|
self.tracker_sock = context.socket(zmq.SUB)
|
|
self.tracker_sock.setsockopt(zmq.CONFLATE, 1) # only keep latest frame. NB. make sure this comes BEFORE connect, otherwise it's ignored!!
|
|
self.tracker_sock.setsockopt(zmq.SUBSCRIBE, b'')
|
|
self.tracker_sock.connect(config.zmq_trajectory_addr)
|
|
|
|
self.frame_sock = context.socket(zmq.SUB)
|
|
self.frame_sock.setsockopt(zmq.CONFLATE, 1) # only keep latest frame. NB. make sure this comes BEFORE connect, otherwise it's ignored!!
|
|
self.frame_sock.setsockopt(zmq.SUBSCRIBE, b'')
|
|
self.frame_sock.connect(config.zmq_frame_addr)
|
|
|
|
|
|
self.H = self.config.H
|
|
|
|
|
|
self.inv_H = np.linalg.pinv(self.H)
|
|
|
|
# TODO: get FPS from frame_emitter
|
|
# self.out = cv2.VideoWriter(str(filename), fourcc, 23.97, (1280,720))
|
|
self.fps = 60
|
|
self.frame_size = (self.config.camera.w,self.config.camera.h)
|
|
self.hide_stats = False
|
|
self.out_writer = self.start_writer() if self.config.render_file else None
|
|
self.streaming_process = self.start_streaming() if self.config.render_url else None
|
|
|
|
self.first_time: float|None = None
|
|
self.frame: Frame|None= None
|
|
self.tracker_frame: Frame|None = None
|
|
self.prediction_frame: Frame|None = None
|
|
|
|
self.tracks: Dict[str, Track] = {}
|
|
self.predictions: Dict[str, Track] = {}
|
|
|
|
|
|
# self.init_shapes()
|
|
|
|
# self.init_labels()
|
|
|
|
|
|
def init_shapes(self):
|
|
'''
|
|
Due to error when running headless, we need to configure options before extending the shapes class
|
|
'''
|
|
class GradientLine(shapes.Line):
|
|
def __init__(self, x, y, x2, y2, width=1, color1=[255,255,255], color2=[255,255,255], batch=None, group=None):
|
|
# print('colors!', colors)
|
|
# assert len(colors) == 6
|
|
|
|
r, g, b, *a = color1
|
|
self._rgba1 = (r, g, b, a[0] if a else 255)
|
|
r, g, b, *a = color2
|
|
self._rgba2 = (r, g, b, a[0] if a else 255)
|
|
|
|
# print('rgba', self._rgba)
|
|
|
|
super().__init__(x, y, x2, y2, width, color1, batch=None, group=None)
|
|
# <pyglet.graphics.vertexdomain.VertexList
|
|
# pyglet.graphics.vertexdomain
|
|
# print(self._vertex_list)
|
|
|
|
def _create_vertex_list(self):
|
|
'''
|
|
copy of super()._create_vertex_list but with additional colors'''
|
|
self._vertex_list = self._group.program.vertex_list(
|
|
6, self._draw_mode, self._batch, self._group,
|
|
position=('f', self._get_vertices()),
|
|
colors=('Bn', self._rgba1+ self._rgba2 + self._rgba2 + self._rgba1 + self._rgba2 +self._rgba1 ),
|
|
translation=('f', (self._x, self._y) * self._num_verts))
|
|
|
|
def _update_colors(self):
|
|
self._vertex_list.colors[:] = self._rgba1+ self._rgba2 + self._rgba2 + self._rgba1 + self._rgba2 +self._rgba1
|
|
|
|
def color1(self, color):
|
|
r, g, b, *a = color
|
|
self._rgba1 = (r, g, b, a[0] if a else 255)
|
|
self._update_colors()
|
|
|
|
def color2(self, color):
|
|
r, g, b, *a = color
|
|
self._rgba2 = (r, g, b, a[0] if a else 255)
|
|
self._update_colors()
|
|
|
|
self.gradientLine = GradientLine
|
|
|
|
def init_labels(self):
|
|
base_color = (255,)*4
|
|
color_predictor = (255,255,0, 255)
|
|
color_info = (255,0, 255, 255)
|
|
color_tracker = (0,255, 255, 255)
|
|
|
|
options = []
|
|
for option in ['prediction_horizon','num_samples','full_dist','gmm_mode','z_mode', 'model_dir']:
|
|
options.append(f"{option}: {self.config.__dict__[option]}")
|
|
|
|
self.labels = {
|
|
'waiting': pyglet.text.Label("Waiting for prediction"),
|
|
'frame_idx': pyglet.text.Label("", x=20, y=self.window.height - 17, color=base_color, batch=self.batch_overlay),
|
|
'tracker_idx': pyglet.text.Label("", x=90, y=self.window.height - 17, color=color_tracker, batch=self.batch_overlay),
|
|
'pred_idx': pyglet.text.Label("", x=110, y=self.window.height - 17, color=color_predictor, batch=self.batch_overlay),
|
|
'frame_time': pyglet.text.Label("t", x=140, y=self.window.height - 17, color=base_color, batch=self.batch_overlay),
|
|
'frame_latency': pyglet.text.Label("", x=235, y=self.window.height - 17, color=color_info, batch=self.batch_overlay),
|
|
'tracker_time': pyglet.text.Label("", x=300, y=self.window.height - 17, color=color_tracker, batch=self.batch_overlay),
|
|
'pred_time': pyglet.text.Label("", x=360, y=self.window.height - 17, color=color_predictor, batch=self.batch_overlay),
|
|
'track_len': pyglet.text.Label("", x=800, y=self.window.height - 17, color=color_tracker, batch=self.batch_overlay),
|
|
'options1': pyglet.text.Label(options.pop(-1), x=20, y=30, color=base_color, batch=self.batch_overlay),
|
|
'options2': pyglet.text.Label(" | ".join(options), x=20, y=10, color=base_color, batch=self.batch_overlay),
|
|
}
|
|
|
|
def refresh_labels(self, dt: float):
|
|
"""Every frame"""
|
|
|
|
if self.frame:
|
|
self.labels['frame_idx'].text = f"{self.frame.index:06d}"
|
|
self.labels['frame_time'].text = f"{self.frame.time - self.first_time: >10.2f}s"
|
|
self.labels['frame_latency'].text = f"{self.frame.time - time.time():.2f}s"
|
|
|
|
if self.tracker_frame:
|
|
self.labels['tracker_idx'].text = f"{self.tracker_frame.index - self.frame.index}"
|
|
self.labels['tracker_time'].text = f"{self.tracker_frame.time - time.time():.3f}s"
|
|
self.labels['track_len'].text = f"{len(self.tracker_frame.tracks)} tracks"
|
|
|
|
if self.prediction_frame:
|
|
self.labels['pred_idx'].text = f"{self.prediction_frame.index - self.frame.index}"
|
|
self.labels['pred_time'].text = f"{self.prediction_frame.time - time.time():.3f}s"
|
|
# self.labels['track_len'].text = f"{len(self.prediction_frame.tracks)} tracks"
|
|
|
|
|
|
# cv2.putText(img, f"{frame.index:06d}", (20,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
# cv2.putText(img, f"{frame.time - first_time:.3f}s", (120,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
|
|
# if prediction_frame:
|
|
# # render Δt and Δ frames
|
|
# cv2.putText(img, f"{prediction_frame.index - frame.index}", (90,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1)
|
|
# cv2.putText(img, f"{prediction_frame.time - time.time():.2f}s", (200,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1)
|
|
# cv2.putText(img, f"{len(prediction_frame.tracks)} tracks", (500,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
# cv2.putText(img, f"h: {np.average([len(t.history or []) for t in prediction_frame.tracks.values()]):.2f}", (580,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1)
|
|
# cv2.putText(img, f"ph: {np.average([len(t.predictor_history or []) for t in prediction_frame.tracks.values()]):.2f}", (660,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1)
|
|
# cv2.putText(img, f"p: {np.average([len(t.predictions or []) for t in prediction_frame.tracks.values()]):.2f}", (740,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1)
|
|
|
|
# options = []
|
|
# for option in ['prediction_horizon','num_samples','full_dist','gmm_mode','z_mode', 'model_dir']:
|
|
# options.append(f"{option}: {config.__dict__[option]}")
|
|
|
|
|
|
# cv2.putText(img, options.pop(-1), (20,img.shape[0]-30), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
# cv2.putText(img, " | ".join(options), (20,img.shape[0]-10), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
|
|
|
|
|
|
def check_frames(self, dt):
|
|
new_tracks = False
|
|
try:
|
|
self.frame: Frame = self.frame_sock.recv_pyobj(zmq.NOBLOCK)
|
|
if not self.first_time:
|
|
self.first_time = self.frame.time
|
|
img = cv2.GaussianBlur(self.frame.img, (15, 15), 0)
|
|
img = cv2.flip(cv2.cvtColor(img, cv2.COLOR_BGR2RGB), 0)
|
|
img = pyglet.image.ImageData(self.frame_size[0], self.frame_size[1], 'RGB', img.tobytes())
|
|
# don't draw in batch, so that it is the background
|
|
self.video_sprite = pyglet.sprite.Sprite(img=img, batch=self.batch_bg)
|
|
self.video_sprite.opacity = 100
|
|
except zmq.ZMQError as e:
|
|
# idx = frame.index if frame else "NONE"
|
|
# logger.debug(f"reuse video frame {idx}")
|
|
pass
|
|
try:
|
|
self.prediction_frame: Frame = self.prediction_sock.recv_pyobj(zmq.NOBLOCK)
|
|
new_tracks = True
|
|
except zmq.ZMQError as e:
|
|
pass
|
|
try:
|
|
self.tracker_frame: Frame = self.tracker_sock.recv_pyobj(zmq.NOBLOCK)
|
|
new_tracks = True
|
|
except zmq.ZMQError as e:
|
|
pass
|
|
|
|
|
|
|
|
def on_key_press(self, symbol, modifiers):
|
|
print('A key was pressed, use f to hide')
|
|
if symbol == ord('f'):
|
|
self.window.set_fullscreen(not self.window.fullscreen)
|
|
if symbol == ord('h'):
|
|
self.hide_stats = not self.hide_stats
|
|
|
|
def check_running(self, dt):
|
|
if not self.is_running.is_set():
|
|
self.window.close()
|
|
self.event_loop.exit()
|
|
|
|
def on_close(self):
|
|
self.is_running.clear()
|
|
|
|
def on_refresh(self, dt: float):
|
|
# update shapes
|
|
# self.bg =
|
|
for track_id, track in self.drawn_tracks.items():
|
|
track.update_drawn_positions(dt)
|
|
|
|
|
|
self.refresh_labels(dt)
|
|
|
|
# self.shape1 = shapes.Circle(700, 150, 100, color=(50, 0, 30), batch=self.batch_anim)
|
|
# self.shape3 = shapes.Circle(800, 150, 100, color=(100, 225, 30), batch=self.batch_anim)
|
|
pass
|
|
|
|
def on_draw(self):
|
|
self.window.clear()
|
|
|
|
self.batch_bg.draw()
|
|
|
|
for track in self.drawn_tracks.values():
|
|
for shape in track.shapes:
|
|
shape.draw() # for some reason the batches don't work
|
|
for track in self.drawn_tracks.values():
|
|
for shapes in track.pred_shapes:
|
|
for shape in shapes:
|
|
shape.draw()
|
|
# self.batch_anim.draw()
|
|
self.batch_overlay.draw()
|
|
|
|
|
|
# pyglet.graphics.draw(3, pyglet.gl.GL_LINE, ("v2i", (100,200, 600,800)), ('c3B', (255,255,255, 255,255,255)))
|
|
|
|
|
|
|
|
if not self.hide_stats:
|
|
self.fps_display.draw()
|
|
|
|
# if streaming, capture buffer and send
|
|
try:
|
|
if self.streaming_process or self.out_writer:
|
|
buf = pyglet.image.get_buffer_manager().get_color_buffer()
|
|
img_data = buf.get_image_data()
|
|
data = img_data.get_data() # alternative: .get_data("RGBA", image_data.pitch)
|
|
img = np.asanyarray(data).reshape((img_data.height, img_data.width, 4))
|
|
img = cv2.cvtColor(img, cv2.COLOR_BGRA2RGB)
|
|
img = np.flip(img, 0)
|
|
# img = cv2.flip(img, cv2.0)
|
|
|
|
# cv2.imshow('frame', img)
|
|
# cv2.waitKey(1)
|
|
if self.streaming_process:
|
|
self.streaming_process.stdin.write(img.tobytes())
|
|
if self.out_writer:
|
|
self.out_writer.write(img)
|
|
except Exception as e:
|
|
logger.exception(e)
|
|
|
|
|
|
def start_writer(self):
|
|
if not self.config.output_dir.exists():
|
|
raise FileNotFoundError("Path does not exist")
|
|
|
|
date_str = datetime.datetime.now().isoformat(timespec="minutes")
|
|
filename = self.config.output_dir / f"render_predictions-{date_str}-{self.config.detector}.mp4"
|
|
logger.info(f"Write to {filename}")
|
|
|
|
return FrameWriter(str(filename), self.fps, self.frame_size)
|
|
|
|
fourcc = cv2.VideoWriter_fourcc(*'vp09')
|
|
|
|
return cv2.VideoWriter(str(filename), fourcc, self.fps, self.frame_size)
|
|
|
|
def start_streaming(self):
|
|
return (
|
|
ffmpeg
|
|
.input('pipe:', format='rawvideo',codec="rawvideo", pix_fmt='bgr24', s='{}x{}'.format(*self.frame_size))
|
|
.output(
|
|
self.config.render_url,
|
|
#codec = "copy", # use same codecs of the original video
|
|
codec='libx264',
|
|
listen=1, # enables HTTP server
|
|
pix_fmt="yuv420p",
|
|
preset="ultrafast",
|
|
tune="zerolatency",
|
|
# g=f"{self.fps*2}",
|
|
g=f"{60*2}",
|
|
analyzeduration="2000000",
|
|
probesize="1000000",
|
|
f='mpegts'
|
|
)
|
|
.overwrite_output()
|
|
.run_async(pipe_stdin=True)
|
|
)
|
|
# return process
|
|
|
|
|
|
|
|
|
|
def run(self, timer_counter):
|
|
frame = None
|
|
prediction_frame = None
|
|
tracker_frame = None
|
|
|
|
i=0
|
|
first_time = None
|
|
|
|
cv2.namedWindow("frame", cv2.WINDOW_NORMAL)
|
|
# https://gist.github.com/ronekko/dc3747211543165108b11073f929b85e
|
|
cv2.moveWindow("frame", 1920, -1)
|
|
cv2.setWindowProperty("frame",cv2.WND_PROP_FULLSCREEN,cv2.WINDOW_FULLSCREEN)
|
|
|
|
while self.is_running.is_set():
|
|
i+=1
|
|
with timer_counter.get_lock():
|
|
timer_counter.value+=1
|
|
|
|
|
|
# zmq_ev = self.frame_sock.poll(timeout=2000)
|
|
# if not zmq_ev:
|
|
# # when no data comes in, loop so that is_running is checked
|
|
# continue
|
|
|
|
try:
|
|
frame: Frame = self.frame_sock.recv_pyobj(zmq.NOBLOCK)
|
|
except zmq.ZMQError as e:
|
|
# idx = frame.index if frame else "NONE"
|
|
# logger.debug(f"reuse video frame {idx}")
|
|
pass
|
|
# else:
|
|
# logger.debug(f'new video frame {frame.index}')
|
|
|
|
|
|
if frame is None:
|
|
# might need to wait a few iterations before first frame comes available
|
|
time.sleep(.1)
|
|
continue
|
|
|
|
try:
|
|
prediction_frame: Frame = self.prediction_sock.recv_pyobj(zmq.NOBLOCK)
|
|
for track_id, track in prediction_frame.tracks.items():
|
|
prediction_id = f"{track_id}-{track.history[-1].frame_nr}"
|
|
self.predictions[prediction_id] = track
|
|
except zmq.ZMQError as e:
|
|
logger.debug(f'reuse prediction')
|
|
|
|
try:
|
|
tracker_frame: Frame = self.tracker_sock.recv_pyobj(zmq.NOBLOCK)
|
|
|
|
for track_id, track in tracker_frame.tracks.items():
|
|
self.tracks[track_id] = track
|
|
except zmq.ZMQError as e:
|
|
logger.debug(f'reuse tracks')
|
|
|
|
if first_time is None:
|
|
first_time = frame.time
|
|
|
|
# img = frame.img
|
|
img = decorate_frame(frame, tracker_frame, prediction_frame, first_time, self.config, self.tracks, self.predictions, self.config.render_clusters, self.counter_listener)
|
|
|
|
logger.debug(f"write frame {frame.time - first_time:.3f}s")
|
|
if self.out_writer:
|
|
self.out_writer.write(img)
|
|
if self.streaming_process:
|
|
self.streaming_process.stdin.write(img.tobytes())
|
|
if self.config.render_window:
|
|
cv2.imshow('frame',cv2.resize(img, (1920, 1080)))
|
|
# cv2.imshow('frame',img)
|
|
cv2.waitKey(1)
|
|
|
|
# clear out old tracks & predictions:
|
|
|
|
for track_id, track in list(self.tracks.items()):
|
|
if get_animation_position(track, frame) == 1:
|
|
self.tracks.pop(track_id)
|
|
for prediction_id, track in list(self.predictions.items()):
|
|
if get_animation_position(track, frame) == 1:
|
|
self.predictions.pop(prediction_id)
|
|
|
|
logger.info('Stopping')
|
|
|
|
# if i>2:
|
|
if self.streaming_process:
|
|
self.streaming_process.stdin.close()
|
|
if self.out_writer:
|
|
self.out_writer.release()
|
|
if self.streaming_process:
|
|
# oddly wrapped, because both close and release() take time.
|
|
logger.info('wait for closing stream')
|
|
self.streaming_process.wait()
|
|
|
|
logger.info('stopped')
|
|
# colorset = itertools.product([0,255], repeat=3) # but remove white
|
|
# colorset = [(0, 0, 0),
|
|
# (0, 0, 255),
|
|
# (0, 255, 0),
|
|
# (0, 255, 255),
|
|
# (255, 0, 0),
|
|
# (255, 0, 255),
|
|
# (255, 255, 0)
|
|
# ]
|
|
colorset = [
|
|
(255,255,100),
|
|
(255,100,255),
|
|
(100,255,255),
|
|
]
|
|
# colorset = [
|
|
# (0,0,0),
|
|
# ]
|
|
|
|
def get_animation_position(track: Track, current_frame: Frame):
|
|
fade_duration = current_frame.camera.fps * 3
|
|
diff = current_frame.index - track.history[-1].frame_nr
|
|
return max(0, min(1, diff / fade_duration))
|
|
# track.history[-1].frame_nr < (current_frame.index - current_frame.camera.fps * 3)
|
|
# track.history[-1].frame_nr < (current_frame.index - current_frame.camera.fps * 3)
|
|
|
|
|
|
|
|
|
|
|
|
# Deprecated
|
|
def decorate_frame(frame: Frame, tracker_frame: Frame, prediction_frame: Frame, first_time: float, config: Namespace, tracks: Dict[str, Track], predictions: Dict[str, Track], as_clusters = True, counter_listener: CounterListerner|None = None) -> np.array:
|
|
# TODO: replace opencv with QPainter to support alpha? https://doc.qt.io/qtforpython-5/PySide2/QtGui/QPainter.html#PySide2.QtGui.PySide2.QtGui.QPainter.drawImage
|
|
# or https://github.com/pygobject/pycairo?tab=readme-ov-file
|
|
# or https://pyglet.readthedocs.io/en/latest/programming_guide/shapes.html
|
|
# and use http://code.astraw.com/projects/motmot/pygarrayimage.html or https://gist.github.com/nkymut/1cb40ea6ae4de0cf9ded7332f1ca0d55
|
|
# or https://api.arcade.academy/en/stable/index.html (supports gradient color in line -- "Arcade is built on top of Pyglet and OpenGL.")
|
|
|
|
undistorted_img = cv2.undistort(frame.img, config.camera.mtx, config.camera.dist, None, config.camera.newcameramtx)
|
|
dst_img = cv2.warpPerspective(undistorted_img,convert_world_space_to_img_space(config.camera.H),(config.camera.w,config.camera.h))
|
|
# dst_img2 = cv2.warpPerspective(undistorted_img,convert_world_space_to_img_space(config.camera.H), None)
|
|
# cv2.imwrite('/home/ruben/suspicion/DATASETS/hof3/camera2.png', dst_img2)
|
|
|
|
overlay = np.zeros(dst_img.shape, np.uint8)
|
|
# Fill image with red color(set each pixel to red)
|
|
overlay[:] = (0, 0, 0)
|
|
|
|
# img = cv2.addWeighted(dst_img, .2, overlay, .3, 0)
|
|
img = dst_img.copy()
|
|
|
|
# all not working:
|
|
# if i == 1:
|
|
# # thanks to GpG for fixing scaling issue: https://stackoverflow.com/a/39668864
|
|
# scale_factor = 1./20 # from 10m to 1000px
|
|
# S = np.array([[scale_factor, 0,0],[0,scale_factor,0 ],[ 0,0,1 ]])
|
|
# new_H = S * self.H * np.linalg.inv(S)
|
|
# warpedFrame = cv2.warpPerspective(img, new_H, (1000,1000))
|
|
# cv2.imwrite(str(self.config.output_dir / "orig.png"), warpedFrame)
|
|
cv2.rectangle(img, (0,0), (img.shape[1],25), (0,0,0), -1)
|
|
|
|
if not tracker_frame:
|
|
cv2.putText(img, f"and track", (650,17), cv2.FONT_HERSHEY_PLAIN, 1, (255,255,0), 1)
|
|
else:
|
|
for track_id, track in tracks.items():
|
|
inv_H = np.linalg.pinv(tracker_frame.H)
|
|
draw_track_projected(img, track, int(track_id), config.camera, convert_world_points_to_img_points)
|
|
|
|
if not prediction_frame:
|
|
cv2.putText(img, f"Waiting for prediction...", (500,17), cv2.FONT_HERSHEY_PLAIN, 1, (255,255,0), 1)
|
|
# continue
|
|
else:
|
|
for track_id, track in predictions.items():
|
|
inv_H = np.linalg.pinv(prediction_frame.H)
|
|
# For debugging:
|
|
# draw_trackjectron_history(img, track, int(track.track_id), convert_world_points_to_img_points)
|
|
anim_position = get_animation_position(track, frame)
|
|
draw_track_predictions(img, track, int(track.track_id)+1, config.camera, convert_world_points_to_img_points, anim_position=anim_position, as_clusters=as_clusters)
|
|
cv2.putText(img, f"{len(track.predictor_history) if track.predictor_history else 'none'}", to_point(track.history[0].get_foot_coords()), cv2.FONT_HERSHEY_COMPLEX, 1, (255,255,255), 1)
|
|
if prediction_frame.maps:
|
|
for i, m in enumerate(prediction_frame.maps):
|
|
map_img = np.ascontiguousarray(np.flipud(np.transpose(m[0], (2, 1, 0))*255), np.uint8)
|
|
cv2.circle(map_img, (10,50), 5, (0,255,0), 2)
|
|
cv2.line(map_img, (10,50), (10+15, 50), (0,0,255), 2)
|
|
cv2.rectangle(map_img, (0,0), (map_img.shape[1]-1, map_img.shape[0]-1), (255,255,255), 1)
|
|
|
|
height, width, _ = map_img.shape
|
|
padding= 50
|
|
y = img.shape[0] - padding - height
|
|
x = width*i
|
|
|
|
if x+width > img.shape[1]:
|
|
break # stop drawing maps when there's a lot of them
|
|
|
|
img[y:y+height,x:x+width] = map_img
|
|
|
|
|
|
|
|
base_color = (255,)*3
|
|
info_color = (255,255,0)
|
|
predictor_color = (255,0,255)
|
|
tracker_color = (0,255,255)
|
|
|
|
cv2.putText(img, f"{frame.index:06d}", (20,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
cv2.putText(img, f"{frame.time - first_time: >10.2f}s", (150,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
cv2.putText(img, f"{frame.time - time.time():.2f}s", (250,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
|
|
if prediction_frame:
|
|
# render Δt and Δ frames
|
|
cv2.putText(img, f"{tracker_frame.index - frame.index}", (90,17), cv2.FONT_HERSHEY_PLAIN, 1, tracker_color, 1)
|
|
cv2.putText(img, f"{prediction_frame.index - frame.index}", (120,17), cv2.FONT_HERSHEY_PLAIN, 1, predictor_color, 1)
|
|
cv2.putText(img, f"{tracker_frame.time - time.time():.2f}s", (310,17), cv2.FONT_HERSHEY_PLAIN, 1, tracker_color, 1)
|
|
cv2.putText(img, f"{prediction_frame.time - time.time():.2f}s", (380,17), cv2.FONT_HERSHEY_PLAIN, 1, predictor_color, 1)
|
|
cv2.putText(img, f"{len(tracker_frame.tracks)} tracks", (620,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
cv2.putText(img, f"h: {np.average([len(t.history or []) for t in prediction_frame.tracks.values()]):.2f}", (700,17), cv2.FONT_HERSHEY_PLAIN, 1, tracker_color, 1)
|
|
cv2.putText(img, f"ph: {np.average([len(t.predictor_history or []) for t in prediction_frame.tracks.values()]):.2f}", (780,17), cv2.FONT_HERSHEY_PLAIN, 1, predictor_color, 1)
|
|
cv2.putText(img, f"p: {np.average([len(t.predictions or []) for t in prediction_frame.tracks.values()]):.2f}", (860,17), cv2.FONT_HERSHEY_PLAIN, 1, predictor_color, 1)
|
|
|
|
options = []
|
|
for option in ['prediction_horizon','num_samples','full_dist','gmm_mode','z_mode', 'model_dir']:
|
|
options.append(f"{option}: {config.__dict__[option]}")
|
|
|
|
|
|
cv2.putText(img, options.pop(-1), (20,img.shape[0]-30), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
cv2.putText(img, " | ".join(options), (20,img.shape[0]-10), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
|
|
# for i, (k, v) in enumerate(counter_listener.get_latest().items()):
|
|
# cv2.putText(img, f"{k} {v.value()}", (20,img.shape[0]-(40*i)-40), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1)
|
|
|
|
|
|
return img
|
|
|
|
|
|
def run_cv_renderer(config: Namespace, is_running: BaseEvent, timer_counter):
|
|
renderer = CvRenderer(config, is_running)
|
|
renderer.run(timer_counter) |