From 6b12ddf08a945808985df34b22a0d56f16cde598 Mon Sep 17 00:00:00 2001 From: Ruben van de Ven Date: Tue, 17 Dec 2024 10:37:30 +0100 Subject: [PATCH] CV renderer and binning options --- trap/animation_renderer.py | 50 ++++++++++++++---- trap/cv_renderer.py | 18 ++++--- trap/frame_emitter.py | 101 ++++++++++++++++++++++--------------- trap/plumber.py | 2 +- trap/prediction_server.py | 3 +- trap/tools.py | 2 +- trap/utils.py | 6 ++- 7 files changed, 120 insertions(+), 62 deletions(-) diff --git a/trap/animation_renderer.py b/trap/animation_renderer.py index 75da0e7..2a8e89b 100644 --- a/trap/animation_renderer.py +++ b/trap/animation_renderer.py @@ -63,13 +63,14 @@ class AnimationRenderer: self.frame_size = (self.config.camera.w,self.config.camera.h) self.hide_stats = False self.out_writer = None # self.start_writer() if self.config.render_file else None - self.streaming_process = None # self.start_streaming() if self.config.render_url else None + self.streaming_process = self.start_streaming() if self.config.render_url else None - if self.config.render_window: - pass - # cv2.namedWindow("frame", cv2.WND_PROP_FULLSCREEN) - # cv2.setWindowProperty("frame",cv2.WND_PROP_FULLSCREEN,cv2.WINDOW_FULLSCREEN) - else: + # if self.config.render_window: + # pass + # # cv2.namedWindow("frame", cv2.WND_PROP_FULLSCREEN) + # # cv2.setWindowProperty("frame",cv2.WND_PROP_FULLSCREEN,cv2.WINDOW_FULLSCREEN) + # else: + if self.streaming_process is not None: pyglet.options["headless"] = True config = pyglet.gl.Config(sample_buffers=1, samples=4) @@ -78,8 +79,10 @@ class AnimationRenderer: display = pyglet.canvas.get_display() screen = display.get_screens()[0] - # self.window = pyglet.window.Window(width=self.frame_size[0], height=self.frame_size[1], config=config, fullscreen=False, screen=screens[1]) - self.window = pyglet.window.Window(width=screen.width, height=screen.height, config=config, fullscreen=True, screen=screen) + if self.streaming_process is not None: + self.window = pyglet.window.Window(width=self.frame_size[0], height=self.frame_size[1], config=config, fullscreen=False, screen=screen) + else: + self.window = pyglet.window.Window(width=screen.width, height=screen.height, config=config, fullscreen=True, screen=screen) self.window.set_handler('on_draw', self.on_draw) self.window.set_handler('on_refresh', self.on_refresh) self.window.set_handler('on_close', self.on_close) @@ -93,8 +96,9 @@ class AnimationRenderer: self.pins = PygletCornerPin( self.window, - source_points=[[540, 670-360], [1380,670-360], [540,760-360], [1380,760-360]], - corners=[[471, 304], [1797, 376], [467, 387], [1792, 484]]) + # source_points=[[540, 670-360], [1380,670-360], [540,760-360], [1380,760-360]], + # corners=[[471, 304], [1797, 376], [467, 387], [1792, 484]] + ) self.window.push_handlers(self.pins) pyglet.gl.glClearColor(255,255,255,255) @@ -153,6 +157,32 @@ class AnimationRenderer: self.init_labels() + def start_streaming(self): + """TODO)) This should be inherited from a generic renderer""" + return ( + ffmpeg + .input('pipe:', format='rawvideo',codec="rawvideo", pix_fmt='bgr24', s='{}x{}'.format(*self.frame_size)) + .output( + self.config.render_url, + #codec = "copy", # use same codecs of the original video + codec='libx264', + listen=1, # enables HTTP server + pix_fmt="yuv420p", + preset="ultrafast", + tune="zerolatency", + # g=f"{self.fps*2}", + g=f"{60*2}", + analyzeduration="2000000", + probesize="1000000", + f='mpegts' + ) + .overwrite_output() + .run_async(pipe_stdin=True) + ) + # return process + + + def init_shapes(self): ''' Due to error when running headless, we need to configure options before extending the shapes class diff --git a/trap/cv_renderer.py b/trap/cv_renderer.py index 535f0b2..ccddfa4 100644 --- a/trap/cv_renderer.py +++ b/trap/cv_renderer.py @@ -483,18 +483,22 @@ def decorate_frame(frame: Frame, tracker_frame: Frame, prediction_frame: Frame, base_color = (255,)*3 info_color = (255,255,0) + predictor_color = (255,0,255) + tracker_color = (0,255,255) cv2.putText(img, f"{frame.index:06d}", (20,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1) - cv2.putText(img, f"{frame.time - first_time:.3f}s", (120,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1) + cv2.putText(img, f"{frame.time - first_time: >10.2f}s", (150,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1) if prediction_frame: # render Δt and Δ frames - cv2.putText(img, f"{prediction_frame.index - frame.index}", (90,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1) - cv2.putText(img, f"{prediction_frame.time - time.time():.2f}s", (200,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1) - cv2.putText(img, f"{len(prediction_frame.tracks)} tracks", (500,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1) - cv2.putText(img, f"h: {np.average([len(t.history or []) for t in prediction_frame.tracks.values()]):.2f}", (580,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1) - cv2.putText(img, f"ph: {np.average([len(t.predictor_history or []) for t in prediction_frame.tracks.values()]):.2f}", (660,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1) - cv2.putText(img, f"p: {np.average([len(t.predictions or []) for t in prediction_frame.tracks.values()]):.2f}", (740,17), cv2.FONT_HERSHEY_PLAIN, 1, info_color, 1) + cv2.putText(img, f"{tracker_frame.index - frame.index}", (90,17), cv2.FONT_HERSHEY_PLAIN, 1, tracker_color, 1) + cv2.putText(img, f"{prediction_frame.index - frame.index}", (120,17), cv2.FONT_HERSHEY_PLAIN, 1, predictor_color, 1) + cv2.putText(img, f"{tracker_frame.time - time.time():.2f}s", (230,17), cv2.FONT_HERSHEY_PLAIN, 1, tracker_color, 1) + cv2.putText(img, f"{prediction_frame.time - time.time():.2f}s", (290,17), cv2.FONT_HERSHEY_PLAIN, 1, predictor_color, 1) + cv2.putText(img, f"{len(tracker_frame.tracks)} tracks", (620,17), cv2.FONT_HERSHEY_PLAIN, 1, base_color, 1) + cv2.putText(img, f"h: {np.average([len(t.history or []) for t in prediction_frame.tracks.values()]):.2f}", (700,17), cv2.FONT_HERSHEY_PLAIN, 1, tracker_color, 1) + cv2.putText(img, f"ph: {np.average([len(t.predictor_history or []) for t in prediction_frame.tracks.values()]):.2f}", (780,17), cv2.FONT_HERSHEY_PLAIN, 1, predictor_color, 1) + cv2.putText(img, f"p: {np.average([len(t.predictions or []) for t in prediction_frame.tracks.values()]):.2f}", (860,17), cv2.FONT_HERSHEY_PLAIN, 1, predictor_color, 1) options = [] for option in ['prediction_horizon','num_samples','full_dist','gmm_mode','z_mode', 'model_dir']: diff --git a/trap/frame_emitter.py b/trap/frame_emitter.py index 6b145ac..5ce7513 100644 --- a/trap/frame_emitter.py +++ b/trap/frame_emitter.py @@ -25,7 +25,8 @@ from bytetracker.basetrack import TrackState as ByteTrackTrackState from trajectron.environment import Environment, Node, Scene from urllib.parse import urlparse -from trap.utils import lerp +from trap.utils import get_bins +from trap.utils import inv_lerp, lerp logger = logging.getLogger('trap.frame_emitter') @@ -219,7 +220,7 @@ class Track: history: List[Detection] = field(default_factory=list) predictor_history: Optional[list] = None # in image space predictions: Optional[list] = None - fps: int = 12 + fps: int = 12 # TODO)) convert this to camera? That way, incorporates H and dist, alternatively, each track is as a whole attached to a space source: Optional[int] = None # to keep track of processed tracks def get_projected_history(self, H: Optional[cv2.Mat] = None, camera: Optional[Camera]= None) -> np.array: @@ -290,49 +291,67 @@ class Track: t.predictions, t.fps/step_size) - # def get_binned(self, bin_size=.5, remove_overlap=True): - # """ - # For an experiment: what if we predict using only concrete positions, by mapping - # dx,dy to a grid. Thus prediction can be for 8 moves, or rather headings - # see ~/notes/attachments example svg - # """ - - # new_history: List[Detection] = [] - # for i, (det0, det1) in enumerate(zip(self.history[:-1], self.history[1:]): - # if i == 0: - # new_history.append(det0) - # continue - # if abs(det1.x - new_history[-1].x) < bin_size or abs(det1.y - new_history[-1].y) < bin_size: - # continue + def get_binned(self, bin_size, camera: Camera, bin_start=True): + """ + For an experiment: what if we predict using only concrete positions, by mapping + dx,dy to a grid. Thus prediction can be for 8 moves, or rather headings + see ~/notes/attachments example svg + """ + + history = self.get_projected_history_as_dict(H=None, camera=camera) + + def round_to_grid_precision(x): + factor = 1/bin_size + return round(x * factor) / factor + + new_history: List[dict] = [] + for i, (det0, det1) in enumerate(zip(history[:-1], history[1:])): + if i == 0: + new_history.append({ + 'x': round_to_grid_precision(det0['x']), + 'y': round_to_grid_precision(det0['y']) + } if bin_start else det0) + continue + if abs(det1['x'] - new_history[-1]['x']) < bin_size and abs(det1['y'] - new_history[-1]['y']) < bin_size: + continue - # # det1 falls outside of the box [-bin_size:+bin_size] around last detection + # det1 falls outside of the box [-bin_size:+bin_size] around last detection - # # 1. Interpolate exact point between det0 and det1 that this happens - # if abs(det1.x - new_history[-1].x) >= bin_size: - # if det1.x - new_history[-1].x >= bin_size: - # # det1 left of last - # x = new_history[-1].x + bin_size - # f = inv_lerp(det0.x, det1.x, x) - # elif new_history[-1].x - det1.x >= bin_size: - # # det1 left of last - # x = new_history[-1].x - bin_size - # f = inv_lerp(det0.x, det1.x, x) - # y = lerp(det0.y, det1.y, f) - # if abs(det1.y - new_history[-1].y) >= bin_size: - # if det1.y - new_history[-1].y >= bin_size: - # # det1 left of last - # y = new_history[-1].y + bin_size - # f = inv_lerp(det0.y, det1.y, x) - # elif new_history[-1].y - det1.y >= bin_size: - # # det1 left of last - # y = new_history[-1].y - bin_size - # f = inv_lerp(det0.y, det1.y, x) - # x = lerp(det0.x, det1.x, f) + # 1. Interpolate exact point between det0 and det1 that this happens + if abs(det1['x'] - new_history[-1]['x']) >= bin_size: + if det1['x'] - new_history[-1]['x'] >= bin_size: + # det1 left of last + x = new_history[-1]['x'] + bin_size + f = inv_lerp(det0['x'], det1['x'], x) + elif new_history[-1]['x'] - det1['x'] >= bin_size: + # det1 left of last + x = new_history[-1]['x'] - bin_size + f = inv_lerp(det0['x'], det1['x'], x) + y = lerp(det0['y'], det1['y'], f) + if abs(det1['y'] - new_history[-1]['y']) >= bin_size: + if det1['y'] - new_history[-1]['y'] >= bin_size: + # det1 left of last + y = new_history[-1]['y'] + bin_size + f = inv_lerp(det0['y'], det1['y'], y) + elif new_history[-1]['y'] - det1['y'] >= bin_size: + # det1 left of last + y = new_history[-1]['y'] - bin_size + f = inv_lerp(det0['y'], det1['y'], y) + x = lerp(det0['x'], det1['x'], f) - # # 2. Find closest point on rectangle (rectangle's four corners, or 4 midpoints) - # points = [[bin_size, 0], [bin_size, bin_size], [0, bin_size], [-bin_size, bin_size], [-bin_size, 0], [-bin_size, -bin_size], [0, -bin_size], [bin_size, -bin_size]] - # # todo Offsets to points:[ history for in points] + # 2. Find closest point on rectangle (rectangle's four corners, or 4 midpoints) + points = get_bins(bin_size) + points = [[new_history[-1]['x']+p[0], new_history[-1]['y'] + p[1]] for p in points] + + distances = [np.linalg.norm([p[0] - x, p[1]-y]) for p in points] + closest = np.argmin(distances) + + point = points[closest] + + new_history.append({'x': point[0], 'y':point[1]}) + # todo Offsets to points:[ history for in points] + return new_history def to_trajectron_node(self, camera: Camera, env: Environment) -> Node: diff --git a/trap/plumber.py b/trap/plumber.py index e1a7b83..71a00a3 100644 --- a/trap/plumber.py +++ b/trap/plumber.py @@ -93,7 +93,7 @@ def start(): ] if args.render_file or args.render_url or args.render_window: - if not args.render_no_preview or args.render_file or args.render_url: + if not args.render_no_preview: #or args.render_file or args.render_url: procs.append( # ExceptionHandlingProcess(target=run_cv_renderer, kwargs={'config': args, 'is_running': isRunning}, name='preview') ExceptionHandlingProcess(target=run_cv_renderer, kwargs={'config': args, 'is_running': isRunning}, name='preview') diff --git a/trap/prediction_server.py b/trap/prediction_server.py index 287908d..53d106e 100644 --- a/trap/prediction_server.py +++ b/trap/prediction_server.py @@ -361,7 +361,8 @@ class PredictionServer: # data = json.dumps({}) # TODO)) signal doing nothing - # self.send_frame(frame) + # TODO)) And update the network + self.send_frame(frame) continue diff --git a/trap/tools.py b/trap/tools.py index 5039b4d..c638480 100644 --- a/trap/tools.py +++ b/trap/tools.py @@ -518,4 +518,4 @@ def normalise_position(data: pd.DataFrame): std = data[['x','y']].std(axis=0) data[['x_norm','y_norm']] = (data[['x','y']] - mu) / std - return data, mu, std \ No newline at end of file + return data, mu, std diff --git a/trap/utils.py b/trap/utils.py index 7b750f8..36e3325 100644 --- a/trap/utils.py +++ b/trap/utils.py @@ -16,4 +16,8 @@ def inv_lerp(a: float, b: float, v: float) -> float: 0.5 == inv_lerp(0, 100, 50) 0.8 == inv_lerp(1, 5, 4.2) """ - return (v - a) / (b - a) \ No newline at end of file + return (v - a) / (b - a) + + +def get_bins(bin_size: float): + return [[bin_size, 0], [bin_size, bin_size], [0, bin_size], [-bin_size, bin_size], [-bin_size, 0], [-bin_size, -bin_size], [0, -bin_size], [bin_size, -bin_size]] \ No newline at end of file