write frames and animation tweaks

pyproject.toml

@@ -47,7 +47,10 @@ process_data = "trap.process_data:main"
 blacklist = "trap.tools:blacklist_tracks"
 rewrite_tracks = "trap.tools:rewrite_raw_track_files"
 
+model_train = "trap.models.train:train"
+
 trap_video_source = "trap.frame_emitter:FrameEmitter.parse_and_start"
+trap_video_writer = "trap.frame_writer:FrameWriter.parse_and_start"
 trap_tracker = "trap.tracker:Tracker.parse_and_start"
 trap_stage = "trap.stage:Stage.parse_and_start"
 trap_prediction = "trap.prediction_server:PredictionServer.parse_and_start"

supervisord.conf

@@ -23,8 +23,8 @@ directory=%(here)s
 autostart=false
 
 [program:video]
-command=uv run trap_video_source  --homography ../DATASETS/hof3/homography.json   --video-src ../DATASETS/hof3/hof3-cam-demo-twoperson.mp4  --calibration ../DATASETS/hof3/calibration.json --video-loop
-# command=uv run trap_video_source  --homography ../DATASETS/hof3-cam-baumer/homography.json   --video-src gige://../DATASETS/hof3-cam-baumer/gige_config.json --calibration ../DATASETS/hof3-cam-baumer/calibration.json
+# command=uv run trap_video_source  --homography ../DATASETS/hof3/homography.json   --video-src ../DATASETS/hof3/hof3-cam-demo-twoperson.mp4  --calibration ../DATASETS/hof3/calibration.json --video-loop
+command=uv run trap_video_source  --homography ../DATASETS/hof3-cam-baumer-cropped/homography.json   --video-src gige://../DATASETS/hof3-cam-baumer-cropped/gige_config.json --calibration ../DATASETS/hof3-cam-baumer-cropped/calibration.json
 directory=%(here)s
 directory=%(here)s
 
@@ -38,6 +38,8 @@ directory=%(here)s
 
 [program:predictor]
 command=uv run trap_prediction --eval_device cuda:0  --model_dir EXPERIMENTS/models/models_20241229_21_35_13_hof3-m2-ud-split-conv12-f2.0-map-2024-12-29/    --num-samples 1      --map_encoding --eval_data_dict EXPERIMENTS/trajectron-data/hof3-m2-ud-split-nostep-conv12-f2.0-map-2024-12-29_val.pkl --prediction-horizon 120 --gmm-mode True --z-mode
+
+#  uv run trajectron_train --continue_training_from EXPERIMENTS/models/models_20241229_21_35_13_hof3-m2-ud-split-conv12-f2.0-map-2024-12-29/ --eval_every 5 --train_data_dict hof3-nostep-conv12-f2.0-map-2024-12-27_train.pkl --eval_data_dict hof3-nostep-conv12-f2.0-map-2024-12-27_val.pkl --offline_scene_graph no --preprocess_workers 8 --log_dir EXPERIMENTS/models --log_tag _hof3-conv12-f2.0-map-2024-12-27 --train_epochs 10 --conf EXPERIMENTS/config.json  --data_dir EXPERIMENTS/trajectron-data --map_encoding
 directory=%(here)s
 
 [program:render_cv]

trap/base.py

@@ -179,6 +179,7 @@ class DistortedCamera(ABC):
         coords = self.project_points(coords, scale)
         return coords
 
+
 class FisheyeCamera(DistortedCamera):
     def __init__(self, dim1, dim2, dim3, K, D, new_K, scaled_K, balance, H, fps):
             # dimensions as per: https://medium.com/@kennethjiang/calibrate-fisheye-lens-using-opencv-part-2-13990f1b157f
@@ -198,8 +199,24 @@ class FisheyeCamera(DistortedCamera):
 
             
             self.map1, self.map2 = cv2.fisheye.initUndistortRectifyMap(self.scaled_K, self.D, self._R, self.new_K, self.dim3, cv2.CV_16SC2)
+            # self.map1, self.map2 = cv2.fisheye.initUndistortRectifyMap(self.scaled_K, self.D, self._R, self.new_K, self.dim3, cv2.CV_32FC1)
             
     def undistort_img(self, img: MatLike):
+        # map1, map2 = adjust_remap_maps(self.map1, self.map2, 2, (0,0))
+        # this only works on the undistort, but screws up when doing subsequent homography,
+        # there needs to be a way to combine both this remap and warpPerspective into a 
+        # single remap call...
+        # scale = 0.3
+        # cx = self.dim3[0] / 2
+        # cy = self.dim3[1] / 2
+
+        # map1 = (self.map1 - cx) / scale + cx
+        # map2 = (self.map2 - cy) / scale + cy
+
+        # map1 += 900  #translate x (>0 left, <0 right)
+        # map2 += 1500  #translate y (>0 up, <0 down)
+
+
         return cv2.remap(img, self.map1, self.map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)
 
     def undistort_points(self, distorted_points: PointList):

trap/cv_renderer.py

@@ -333,6 +333,8 @@ def decorate_frame(frame: Frame, tracker_frame: Frame, prediction_frame: Frame,
             points = frame.camera.points_img_to_world(points, scale)
             points = [to_point(p) for p in points] # to int
             
+            w = points[1][0]-points[2][0]
+            feet = [int(points[2][0] + .5 * w), points[2][1]]
             cv2.rectangle(img, points[1],  points[2], (255,255,0), 2)
             cv2.circle(img,  points[0], 5, (255,255,0), 2)
 

trap/frame_emitter.py

@@ -30,8 +30,6 @@ class FrameEmitter(node.Node):
 
         self.video_srcs = self.config.video_src
 
-    
-
 
     def run(self):
         offset = int(self.config.video_offset or 0)

trap/frame_writer.py

@@ -0,0 +1,97 @@
+# used for "Forward Referencing of type annotations"
+from __future__ import annotations
+
+import datetime
+import logging
+import time
+from argparse import ArgumentParser
+from pathlib import Path
+
+import zmq
+
+from trap.frame_emitter import Frame
+from trap.node import Node
+from trap.preview_renderer import FrameWriter as CvFrameWriter
+
+logger = logging.getLogger("trap.simple_renderer")
+
+class FrameWriter(Node):
+    def setup(self):
+        self.frame_sock = self.sub(self.config.zmq_frame_addr)
+
+        self.out_writer = self.start_writer()
+
+    def start_writer(self):
+        if not self.config.output_dir.exists():
+            raise FileNotFoundError("Path does not exist")
+    
+        date_str = datetime.datetime.now().isoformat(timespec="minutes")
+        filename = self.config.output_dir / f"render-source-{date_str}.mp4"
+        logger.info(f"Write to {filename}")
+
+        return CvFrameWriter(str(filename), None, None)
+
+        # fourcc = cv2.VideoWriter_fourcc(*'vp09')
+
+        # return cv2.VideoWriter(str(filename), fourcc, self.fps, self.frame_size)
+
+    def run(self):
+        i=0
+        try:
+            while self.run_loop():
+                i += 1
+
+                # zmq_ev = self.frame_sock.poll(timeout=2000)
+                # if not zmq_ev:
+                #     # when no data comes in, loop so that is_running is checked
+                #     continue
+            
+                try:
+                    frame: Frame = self.frame_sock.recv_pyobj(zmq.NOBLOCK)
+                
+                    
+                    # else:
+                    #     logger.debug(f'new video frame  {frame.index}')
+
+                    
+                    if frame is None:
+                        # might need to wait a few iterations before first frame comes available
+                        time.sleep(.1)
+                        continue
+                    
+                    self.logger.debug(f"write frame {frame.time:.3f}")
+                    self.out_writer.write(frame.img)
+
+                except zmq.ZMQError as e:
+                    # idx = frame.index if frame else "NONE"
+                    # logger.debug(f"reuse video frame {idx}")
+                    
+                    pass
+        except KeyboardInterrupt as e:
+            print('stopping on interrupt')
+                    
+        self.logger.info('Stopping')
+
+        # if i>2:
+        if self.out_writer:
+            self.out_writer.release() 
+            self.logger.info(f'Wrote to {self.out_writer.filename}')
+        
+        self.logger.info('stopped')
+
+
+    @classmethod
+    def arg_parser(cls):
+        argparser = ArgumentParser()
+        argparser.add_argument('--zmq-frame-addr',
+                    help='Manually specity communication addr for the frame messages',
+                    type=str,
+                    default="ipc:///tmp/feeds_frame") 
+        
+        argparser.add_argument("--output-dir",
+                    help="Directory to save the video in",
+                    required=True,
+                    type=Path)
+        return argparser
+        
+    

trap/prediction_server.py

@@ -86,7 +86,8 @@ def get_maps_for_input(input_dict, scene, hyperparams, device):
 
             scene_map = scene.map[node.type]
             # map_point = x[-1, :2]
-            map_point = x[:2]
+            # map_point = x[:2]
+            map_point = x[:2].clip(0) # prevent crash for out of map point.
 
             patch_size = hyperparams['map_encoder'][node.type]['patch_size']
 
@@ -102,6 +103,7 @@ def get_maps_for_input(input_dict, scene, hyperparams, device):
         heading_angles = torch.Tensor(heading_angles)
 
     # print(scene_maps, patch_sizes, heading_angles)
+    # print(scene_pts)
     maps = scene_maps[0].get_cropped_maps_from_scene_map_batch(scene_maps,
                                                                scene_pts=torch.Tensor(scene_pts),
                                                                patch_size=patch_sizes[0],

trap/process_data.py

@@ -163,6 +163,8 @@ def process_data(src_dir: Path, dst_dir: Path, name: str, smooth_tracks: bool, c
 
     print(f"Camera FPS: {camera.fps}, actual fps: {camera.fps/step_size} (or {(1/camera.fps)*step_size})")
 
+    names = {}
+
     for data_class, nr_of_items in destinations.items():
         env = Environment(node_type_list=['PEDESTRIAN'], standardization=standardization)
         attention_radius = dict()
@@ -172,6 +174,7 @@ def process_data(src_dir: Path, dst_dir: Path, name: str, smooth_tracks: bool, c
         scenes = []
         split_id = f"{name}_{data_class}"
         data_dict_path = dst_dir / (split_id + '.pkl')
+        names[data_class] = data_dict_path
         # subpath = src_dir / data_class
 
 
@@ -298,6 +301,7 @@ def process_data(src_dir: Path, dst_dir: Path, name: str, smooth_tracks: bool, c
     # print(f"Linear: {l}")
     # print(f"Non-Linear: {nl}")
     print(f"error: {skipped_for_error}, used: {created}")
+    return names
 
 def main():
     parser = argparse.ArgumentParser()

trap/stage.py

@@ -13,7 +13,9 @@ import time
 from typing import Dict, List, Optional, Tuple
 from matplotlib.pyplot import isinteractive
 import numpy as np
-from shapely import LineString, line_locate_point, linestrings
+from shapely import LineString,  MultiLineString, line_locate_point, linestrings
+
+from shapely.ops import substring
 from statemachine import Event, State, StateMachine
 from statemachine.exceptions import TransitionNotAllowed
 import zmq
@@ -36,8 +38,9 @@ logger = logging.getLogger('trap.stage')
 Coordinate = Tuple[float, float]
 DeltaT = float # delta_t in seconds
 
+OPTION_POSITION_MARKER = False
 OPTION_GROW_ANOMALY_CIRCLE = False
-OPTION_RENDER_DIFF_SEGMENT = True
+# OPTION_RENDER_DIFF_SEGMENT = True
 
 class LineGenerator(ABC):
     @abstractmethod
@@ -328,7 +331,7 @@ class ScenarioScene(Enum):
     LOST = -1
 
 LOST_FADEOUT = 3
-PREDICTION_INTERVAL: float|None = 20 # frames
+PREDICTION_INTERVAL: float|None = 10 # frames
 PREDICTION_FADE_IN: float = 3
 PREDICTION_FADE_SLOPE: float = -10
 PREDICTION_FADE_AFTER_DURATION: float = 10 # seconds
@@ -576,8 +579,8 @@ class DrawnScenario(TrackScenario):
         # 0. Update anomaly, slowly decreasing it over time
         self.decay_anomaly_score(dt)
 
-        for diff in self.prediction_diffs:
-            diff.update_drawn_positions(dt, self)
+        # for diff in self.prediction_diffs:
+        #     diff.update_drawn_positions(dt, self)
 
         # 1. track history, direct update
         
@@ -587,8 +590,8 @@ class DrawnScenario(TrackScenario):
         if self.drawn_position is None:
             self.drawn_position = self.drawn_positions[-1]
         else:
-            self.drawn_position[0] = exponentialDecay(self.drawn_position[0], self.drawn_positions[-1][0], 3, dt)
-            self.drawn_position[1] = exponentialDecay(self.drawn_position[1], self.drawn_positions[-1][1], 3, dt)
+            self.drawn_position[0] = exponentialDecay(self.drawn_position[0], self.drawn_positions[-1][0], 13, dt)
+            self.drawn_position[1] = exponentialDecay(self.drawn_position[1], self.drawn_positions[-1][1], 13, dt)
         
         # 3. predictions
         if len(self.drawn_predictions) < len(self.predictions):
@@ -720,36 +723,46 @@ class DrawnScenario(TrackScenario):
 
         # 2. Position Marker / anomaly score
 
-        anomaly_marker_color = SrgbaColor(0.,0.,1, 1.-self.lost_factor()) # fadeout
-        # lines.append(circle_arc(self.drawn_positions[-1][0], self.drawn_positions[-1][1], 1, t, self.anomaly_score, anomaly_marker_color))
-        # last point, (but this draws line in circle, requiring a 'jump back' for the laser)
-        cx, cy = self.drawn_positions[-1][0], self.drawn_positions[-1][1],
+        if OPTION_POSITION_MARKER:
+            anomaly_marker_color = SrgbaColor(0.,0.,1, 1.-self.lost_factor()) # fadeout
+            # lines.append(circle_arc(self.drawn_positions[-1][0], self.drawn_positions[-1][1], 1, t, self.anomaly_score, anomaly_marker_color))
+            # last point, (but this draws line in circle, requiring a 'jump back' for the laser)
+            cx, cy = self.drawn_position[0], self.drawn_position[1],
 
-        radius = max(.1, self._drawn_anomaly_score * 1.) if OPTION_GROW_ANOMALY_CIRCLE else .1
+            radius = max(.1, self._drawn_anomaly_score * 1.) if OPTION_GROW_ANOMALY_CIRCLE else .1
 
-        steps=5
-        if len(self.drawn_positions) >= steps:
-            dx, dy = self.drawn_positions[-1][0] - self.drawn_positions[-steps][0], self.drawn_positions[-1][1] - self.drawn_positions[-steps][1],
-            diff = np.array([dx,dy])
-            diff = diff/np.linalg.norm(diff) * radius * 1.1
-            cx += diff[0]
-            cy += diff[1]
+            steps=0
+            if len(self.drawn_positions) >= steps:
+                dx, dy = self.drawn_positions[-1][0] - self.drawn_positions[-steps][0], self.drawn_positions[-1][1] - self.drawn_positions[-steps][1],
+                diff = np.array([dx,dy])
+                diff = diff/np.linalg.norm(diff) * radius * 1.1
+                cx += diff[0]
+                cy += diff[1]
 
-        lines.append(circle_arc(
-            cx, cy,
-            radius,
-            0, 1,
-            anomaly_marker_color)
-            )
+            lines.append(circle_arc(
+                cx, cy,
+                radius,
+                0, 1,
+                anomaly_marker_color)
+                )
 
         # 3. Predictions
         if len(self.drawn_predictions):
             color = SrgbaColor(0.,1,0.,1.-self.lost_factor())
-            prediction_track_age = time.time() - self.predictions[0].created_at
-            t_factor = prediction_track_age / PREDICTION_FADE_IN
             # positions = [RenderablePosition.from_list(pos) for pos in self.drawn_positions]
             for a, drawn_prediction in enumerate(self.drawn_predictions):
-
+                if a < (len(self.drawn_predictions) - 1):
+                    # not the newest: fade out:
+                    deprecation_age = t - self.predictions[a+1].created_at
+                    if deprecation_age > PREDICTION_FADE_IN:
+                        # old: skip drawing.
+                        continue 
+                    else: 
+                        fade_factor = 1 - (deprecation_age / PREDICTION_FADE_IN)
+                        color = color.as_faded(fade_factor)
+                    
+                prediction_track_age = time.time() - self.predictions[a].created_at
+                t_factor = prediction_track_age / PREDICTION_FADE_IN
 
                 associated_diff = self.prediction_diffs[a]
                 progress = associated_diff.nr_of_passed_points()
@@ -772,7 +785,16 @@ class DrawnScenario(TrackScenario):
                 # points = [RenderablePoint(pos, pos_color) for pos, pos_color in zip(drawn_prediction[PREDICTION_OFFSET:], colors[PREDICTION_OFFSET:])]
                 points = [RenderablePoint(pos, pos_color) for pos, pos_color in zip(drawn_prediction, colors)]
                 points = points[progress//2:]
-                lines.append(RenderableLine(points))
+                ls = LineString(drawn_prediction)
+                if t_factor < 1:
+                    ls = substring(ls, 0, t_factor*ls.length, ls.length)
+                dashed = dashed_line(ls, 1, .5, t)
+                # print(dashed)
+                for line in dashed.geoms:
+                    dash_points = [RenderablePoint(point, color) for point in line.coords]
+                    lines.append(RenderableLine(dash_points))
+                
+                # lines.append(RenderableLine(points))
 
         # 4. Diffs
         # for drawn_diff in self.drawn_diffs:
@@ -781,9 +803,10 @@ class DrawnScenario(TrackScenario):
         #     points = [RenderablePoint(pos, pos_color) for pos, pos_color in zip(drawn_diff, colors)]
         #     lines.append(RenderableLine(points))
 
-        if OPTION_RENDER_DIFF_SEGMENT:
-            for diff in self.prediction_diffs:
-                lines.append_lines(diff.as_renderable())
+        # if OPTION_RENDER_DIFF_SEGMENT:
+        #     for diff in self.prediction_diffs:
+        #         lines.append_lines(diff.as_renderable())
+        #         pass
 
 
         # # print(self.current_state)
@@ -1001,7 +1024,7 @@ class Stage(Node):
 
 # TODO place somewhere else:
 # Gemma3:27b prompt: "python. Given a list of coordinates, that describes a line: `drawable_points: List[Tuple[float,float]]` apply perlin noise over the normal of the line, that changes over time `dt`."
-def apply_perlin_noise_to_line_normal(drawable_points: np.ndarray, dt: float, amplitude: float = 1.0, frequency: float = 1.0) -> np.ndarray:
+def apply_perlin_noise_to_line_normal(drawable_points: np.ndarray, dt: float, amplitude: float = 1.0, frequency: float = 1.0, fade_over_n_points = 8) -> np.ndarray:
     """
     Applies Perlin noise to the normals of a line described by a list of coordinates, changing over time.
 
@@ -1067,9 +1090,16 @@ def apply_perlin_noise_to_line_normal(drawable_points: np.ndarray, dt: float, am
         # noise_x = noise([x * frequency, (y + dt) * frequency]) * amplitude * normal_x
         # noise_y = noise([x * frequency, (y + dt) * frequency]) * amplitude * normal_y
         noise = snoise2(i * frequency, dt % 1000, octaves=4)
+
+        use_amp = amplitude
+        if fade_over_n_points > 0:
+            rev_step = len(drawable_points) - i
+            amp_factor = rev_step / fade_over_n_points
+            if amp_factor < 1:
+                use_amp *= amp_factor
         
-        noise_x = noise * amplitude * normal_x
-        noise_y = noise * amplitude * normal_y
+        noise_x = noise * use_amp * normal_x
+        noise_y = noise * use_amp * normal_y
 
         # print(noise_x, noise_y, dt, frequency, i, dt, snoise2(i * frequency, dt % 1000, octaves=4))
         
@@ -1082,4 +1112,29 @@ def apply_perlin_noise_to_line_normal(drawable_points: np.ndarray, dt: float, am
     
     # print(drawable_points, new_points)
 
-    return np.array(new_points)
+    return np.array(new_points)
+
+
+import math
+
+def distance(p1, p2):
+    return math.hypot(p2[0] - p1[0], p2[1] - p1[1])
+
+
+def dashed_line(line: LineString, dash_len: float, gap_len: float, offset: float = 0) -> MultiLineString:
+    total_length = line.length
+
+    segments = []
+    pos = offset % (dash_len + gap_len)
+
+    if pos > gap_len:
+        segments.append(substring(line, 0, pos - gap_len))
+
+    while pos < total_length:
+        end = min(pos + dash_len, total_length)
+        if pos < end:
+            dash = substring(line, pos, end)
+            segments.append(dash)
+        pos += dash_len + gap_len
+
+    return MultiLineString(segments)

trap/tracker.py

@@ -443,9 +443,11 @@ class Tracker(Node):
             # self.model = YOLO('EXPERIMENTS/yolov8x.pt')
             # best from arsen:
             # self.model = YOLO('./tracker/all_yolo11-2-20-15-41/weights')
+            # self.model = YOLO('tracker/all_yolo11-2-20-15-41/weights/best.pt')
             # self.model = YOLO('models/yolo11x-pose.pt')
             # self.model = YOLO("models/yolo12l.pt") 
-            self.model = YOLO("models/yolo12x.pt") 
+            # self.model = YOLO("models/yolo12x.pt", imgsz=self.config.imgsz) #see https://github.com/orgs/ultralytics/discussions/8812
+            self.model = YOLO("models/yolo12x.pt")
             # NOTE: changing the model, also tweak imgsz in 
         elif self.config.detector == DETECTOR_RTDETR:
             # self.model = RTDETR('models/rtdetr-x.pt') # drops frames
@@ -724,7 +726,7 @@ class Tracker(Node):
         argparser.add_argument("--imgsz",
                             help="Detector imgsz parameter (applicable to ultralytics detectors)",
                             type=int,
-                            default=960)
+                            default=480)
         return argparser
 
 

trap/video_sources.py

@@ -35,6 +35,14 @@ class GigEConfig:
     binning_v: BinningValue = 1
     pixel_format: int = neoapi.PixelFormat_BayerRG8
 
+    # when changing these values, make sure you also tweak the calibration
+    width: int = 2448
+    height: int = 2048
+
+    # changing these _automatically changes calibration cx and cy_!!
+    offset_x: int = 0
+    offset_y: int = 0
+
     post_crop_tl: Optional[Coordinate] = None
     post_crop_br: Optional[Coordinate] = None
 
@@ -58,47 +66,90 @@ class GigE(VideoSource):
         self.camera.SetImageBufferCycleCount(1) 
         self.setPixelFormat(self.config.pixel_format)
 
+        self.cam_is_configured = False
+
+        self.converter_settings = neoapi.ConverterSettings()
+        self.converter_settings.SetDebayerFormat('BGR8') # opencv
+        self.converter_settings.SetDemosaicingMethod(neoapi.ConverterSettings.Demosaicing_Baumer5x5)
+        # self.converter_settings.SetSharpeningMode(neoapi.ConverterSettings.Sharpening_Global)
+        # self.converter_settings.SetSharpeningMode(neoapi.ConverterSettings.Sharpening_ActiveNoiseReduction)
+        # self.converter_settings.SetSharpeningFactor(3)
+        # self.converter_settings.SetSharpeningSensitivityThreshold(2)
+        
+
+        
+    
+    def configCam(self):
         if self.camera.IsConnected(): 
+            self.setPixelFormat(self.config.pixel_format)
+
             # self.camera.f.PixelFormat.Set(neoapi.PixelFormat_RGB8)
             self.camera.f.BinningHorizontal.Set(self.config.binning_h)
             self.camera.f.BinningVertical.Set(self.config.binning_v)
+            self.camera.f.Height.Set(self.config.height)
+            self.camera.f.Width.Set(self.config.width)
+            self.camera.f.OffsetX.Set(self.config.offset_x)
+            self.camera.f.OffsetY.Set(self.config.offset_y)
+
             # print('exposure time', self.camera.f.ExposureAutoMaxValue.Set(20000)) # shutter 1/50
             print('exposure time', self.camera.f.ExposureAutoMaxValue.Set(25000))
             print('brightness targt', self.camera.f.BrightnessAutoNominalValue.Get())
             print('brightness targt', self.camera.f.BrightnessAutoNominalValue.Set(30))
             print('exposure time', self.camera.f.ExposureTime.Get())
             print('Gamma', self.camera.f.Gamma.Set(0.39))
+
+            # neoapi.region
+            # self.camera.f.regeo
             # print('LUT', self.camera.f.LUTIndex.Get())
             # print('LUT', self.camera.f.LUTEnable.Get())
             # print('exposure time max', self.camera.f.ExposureTimeGapMax.Get())
             # print('exposure time min', self.camera.f.ExposureTimeGapMin.Get())
         # self.pixfmt = self.camera.f.PixelFormat.Get()
+
+        self.cam_is_configured = True
     
     def setPixelFormat(self, pixfmt):
         self.pixfmt = pixfmt
         self.camera.f.PixelFormat.Set(pixfmt)
         # self.pixfmt = self.camera.f.PixelFormat.Get()
     
+    
     def recv(self):
         while True:
+            # print('receive')
             if not self.camera.IsConnected():
+                self.cam_is_configured = False
                 return
+
+            if not self.cam_is_configured:
+                self.configCam()
+            
+
             
             i = self.camera.GetImage(0)
             if i.IsEmpty():
                 time.sleep(.01)
                 continue
 
-            imgarray = i.GetNPArray()
-            if self.pixfmt == neoapi.PixelFormat_BayerRG12:
-                img = cv2.cvtColor(imgarray, cv2.COLOR_BayerRG2RGB)
-            elif self.pixfmt == neoapi.PixelFormat_BayerRG8:
-                img = cv2.cvtColor(imgarray, cv2.COLOR_BayerRG2RGB)
-            else:
-                img = cv2.cvtColor(imgarray, cv2.COLOR_BGR2RGB)
+            # print(i.GetAvailablePixelFormats())
+            i = i.Convert(self.converter_settings)
             
-            if img.dtype == np.uint16:
-                img = cv2.convertScaleAbs(img, alpha=(255.0/65535.0))
+            if i.IsEmpty():
+                time.sleep(.01)
+                continue
+
+            img = i.GetNPArray()
+
+            # imgarray = i.GetNPArray()
+            # if self.pixfmt == neoapi.PixelFormat_BayerRG12:
+            #     img = cv2.cvtColor(imgarray, cv2.COLOR_BayerRG2RGB)
+            # elif self.pixfmt == neoapi.PixelFormat_BayerRG8:
+            #     img = cv2.cvtColor(imgarray, cv2.COLOR_BayerRG2RGB)
+            # else:
+            #     img = cv2.cvtColor(imgarray, cv2.COLOR_BGR2RGB)
+            
+            # if img.dtype == np.uint16:
+            #     img = cv2.convertScaleAbs(img, alpha=(255.0/65535.0))
             img = self._crop(img)
             yield img