From 531d61b69aaedbeac91b1889985e8c4157f1a21a Mon Sep 17 00:00:00 2001
From: Ruben van de Ven <git@rubenvandeven.com>
Date: Thu, 20 Jun 2024 12:32:05 +0200
Subject: [PATCH] detach render FPS from frame emit

---
 trap/frame_emitter.py |  3 ++-
 trap/renderer.py      | 37 ++++++++++++++++++++++++++-----------
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/trap/frame_emitter.py b/trap/frame_emitter.py
index abe1232..1523084 100644
--- a/trap/frame_emitter.py
+++ b/trap/frame_emitter.py
@@ -148,10 +148,11 @@ class FrameEmitter:
             if str(video_path).isdigit():
                 # numeric input is a CV camera
                 video = cv2.VideoCapture(int(str(video_path)))
+                # TODO: make config variables
                 video.set(cv2.CAP_PROP_FRAME_WIDTH, int(1280))
                 video.set(cv2.CAP_PROP_FRAME_HEIGHT, int(720))
                 print("exposure!", video.get(cv2.CAP_PROP_AUTO_EXPOSURE))
-                video.set(cv2.CAP_PROP_FPS, 1)
+                video.set(cv2.CAP_PROP_FPS, 5)
             else:
                 video = cv2.VideoCapture(str(video_path))
             fps = video.get(cv2.CAP_PROP_FPS)
diff --git a/trap/renderer.py b/trap/renderer.py
index 710d9b6..cbfb710 100644
--- a/trap/renderer.py
+++ b/trap/renderer.py
@@ -125,18 +125,33 @@ class Renderer:
 
     
     def run(self):
+        frame = None
         prediction_frame = None
+
         i=0
         first_time = None
         while self.is_running.is_set():
             i+=1
 
-            zmq_ev = self.frame_sock.poll(timeout=2000)
-            if not zmq_ev:
-                # when no data comes in, loop so that is_running is checked
-                continue
 
-            frame: Frame = self.frame_sock.recv_pyobj()
+            # zmq_ev = self.frame_sock.poll(timeout=2000)
+            # if not zmq_ev:
+            #     # when no data comes in, loop so that is_running is checked
+            #     continue
+        
+            try:
+                frame: Frame = self.frame_sock.recv_pyobj(zmq.NOBLOCK)
+            except zmq.ZMQError as e:
+                idx = frame.index if frame else "NONE"
+                logger.debug(f"reuse video frame {idx}")
+            else:
+                logger.debug(f'new video frame  {frame.index}')
+            
+            if frame is None:
+                # might need to wait a few iterations before first frame comes available
+                time.sleep(.1)
+                continue
+            
             try:
                 prediction_frame: Frame = self.prediction_sock.recv_pyobj(zmq.NOBLOCK)
             except zmq.ZMQError as e:
@@ -145,7 +160,7 @@ class Renderer:
             if first_time is None:
                     first_time = frame.time
 
-            decorate_frame(frame, prediction_frame, first_time, self.config)
+            img = decorate_frame(frame, prediction_frame, first_time, self.config)
 
             img_path = (self.config.output_dir / f"{i:05d}.png").resolve()
 
@@ -153,11 +168,11 @@ class Renderer:
             
             logger.debug(f"write frame {frame.time - first_time:.3f}s")
             if self.out_writer:
-                self.out_writer.write(frame.img)
+                self.out_writer.write(img)
             if self.streaming_process:
-                self.streaming_process.stdin.write(frame.img.tobytes())
+                self.streaming_process.stdin.write(img.tobytes())
             if self.config.render_window:
-                cv2.imshow('frame',frame.img)
+                cv2.imshow('frame',img)
                 cv2.waitKey(1)
         logger.info('Stopping')
 
@@ -188,8 +203,8 @@ def decorate_frame(frame: Frame, prediction_frame: Frame, first_time: float, con
     # Fill image with red color(set each pixel to red)
     overlay[:] = (130, 0, 75)
 
-    frame.img = cv2.addWeighted(frame.img, .4, overlay, .6, 0)
-    img = frame.img
+    img = cv2.addWeighted(frame.img, .4, overlay, .6, 0)
+    # img = frame.img.copy()
 
     # all not working:
     # if i == 1: