From 4c126876b1d2761e5ad9b926ef32255fc1771481 Mon Sep 17 00:00:00 2001 From: Ruben van de Ven Date: Mon, 12 May 2025 17:36:53 +0200 Subject: [PATCH] fix non rendering lines --- test_training.ipynb | 61 +++++++++++++++++++++++++-------------------- trap/base.py | 52 +++++++++++++++++++++++--------------- trap/cv_renderer.py | 3 ++- trap/tracker.py | 3 ++- 4 files changed, 70 insertions(+), 49 deletions(-) diff --git a/test_training.ipynb b/test_training.ipynb index 90893a0..940558a 100644 --- a/test_training.ipynb +++ b/test_training.ipynb @@ -36,21 +36,15 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/ruben/suspicion/trap/.venv/lib/python3.10/site-packages/torch/cuda/__init__.py:83: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at ../c10/cuda/CUDAFunctions.cpp:109.)\n", - " return torch._C._cuda_getDeviceCount() > 0\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Training options\n", + "from trap.base import DistortedCamera\n", + "\n", + "\n", "input_seq_length = 36\n", "output_seq_length = 36\n", "\n", @@ -72,7 +66,7 @@ "calibration_path = Path(\"../DATASETS/hof3/calibration.json\")\n", "homography_path = Path(\"../DATASETS/hof3/homography.json\")\n", "device = device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", - "camera = Camera.from_paths(calibration_path, homography_path, 12)\n", + "camera = DistortedCamera.from_paths(calibration_path, homography_path, 12)\n", "\n", "snapshot_path = Path(\"../DATASETS/hof3/output.png\")\n", "\n", @@ -88,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -109,7 +103,7 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[14], line 17\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28mprint\u001b[39m(coords[:,\u001b[38;5;241m0\u001b[39m])\n\u001b[1;32m 16\u001b[0m plt\u001b[38;5;241m.\u001b[39mimshow(dst_img, extent\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m0\u001b[39m, dst_img\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m]\u001b[38;5;241m/\u001b[39m\u001b[38;5;241m100\u001b[39m, dst_img\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m/\u001b[39m\u001b[38;5;241m100\u001b[39m,\u001b[38;5;241m0\u001b[39m)),\n\u001b[0;32m---> 17\u001b[0m \u001b[43mplt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mplot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcoords\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcoords\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mro\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", + "Cell \u001b[0;32mIn[3], line 17\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28mprint\u001b[39m(coords[:,\u001b[38;5;241m0\u001b[39m])\n\u001b[1;32m 16\u001b[0m plt\u001b[38;5;241m.\u001b[39mimshow(dst_img, extent\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m0\u001b[39m, dst_img\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m]\u001b[38;5;241m/\u001b[39m\u001b[38;5;241m100\u001b[39m, dst_img\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m/\u001b[39m\u001b[38;5;241m100\u001b[39m,\u001b[38;5;241m0\u001b[39m)),\n\u001b[0;32m---> 17\u001b[0m \u001b[43mplt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mplot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcoords\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcoords\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mro\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/suspicion/trap/.venv/lib/python3.10/site-packages/matplotlib/pyplot.py:2767\u001b[0m, in \u001b[0;36mplot\u001b[0;34m(scalex, scaley, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m 2765\u001b[0m \u001b[38;5;129m@_copy_docstring_and_deprecators\u001b[39m(Axes\u001b[38;5;241m.\u001b[39mplot)\n\u001b[1;32m 2766\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mplot\u001b[39m(\u001b[38;5;241m*\u001b[39margs, scalex\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, scaley\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m-> 2767\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mgca\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mplot\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2768\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mscalex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mscalex\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mscaley\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mscaley\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2769\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdata\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m}\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/suspicion/trap/.venv/lib/python3.10/site-packages/matplotlib/axes/_axes.py:1635\u001b[0m, in \u001b[0;36mAxes.plot\u001b[0;34m(self, scalex, scaley, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1393\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1394\u001b[0m \u001b[38;5;124;03mPlot y versus x as lines and/or markers.\u001b[39;00m\n\u001b[1;32m 1395\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1632\u001b[0m \u001b[38;5;124;03m(``'green'``) or hex strings (``'#008000'``).\u001b[39;00m\n\u001b[1;32m 1633\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1634\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m cbook\u001b[38;5;241m.\u001b[39mnormalize_kwargs(kwargs, mlines\u001b[38;5;241m.\u001b[39mLine2D)\n\u001b[0;32m-> 1635\u001b[0m lines \u001b[38;5;241m=\u001b[39m [\u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_lines(\u001b[38;5;241m*\u001b[39margs, data\u001b[38;5;241m=\u001b[39mdata, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)]\n\u001b[1;32m 1636\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m line \u001b[38;5;129;01min\u001b[39;00m lines:\n\u001b[1;32m 1637\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39madd_line(line)\n", "File \u001b[0;32m~/suspicion/trap/.venv/lib/python3.10/site-packages/matplotlib/axes/_base.py:312\u001b[0m, in \u001b[0;36m_process_plot_var_args.__call__\u001b[0;34m(self, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m 310\u001b[0m this \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m args[\u001b[38;5;241m0\u001b[39m],\n\u001b[1;32m 311\u001b[0m args \u001b[38;5;241m=\u001b[39m args[\u001b[38;5;241m1\u001b[39m:]\n\u001b[0;32m--> 312\u001b[0m \u001b[38;5;28;01myield from\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_plot_args\u001b[49m\u001b[43m(\u001b[49m\u001b[43mthis\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", @@ -157,7 +151,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -167,7 +161,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -193,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -202,21 +196,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "reader = TrackReader(path, camera.fps, exclude_whitelisted = False, include_blacklisted=False)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 25, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "actual 26 projected 1\n" + ] } ], - "source": [] + "source": [ + "\n", + "track = reader._tracks[\"48\"]\n", + "\n", + "track = track.get_with_interpolated_history()\n", + "history = track.get_projected_history(H=None, camera=camera)\n", + "print(\"actual\", len(track.history), \"projected\", len(history))" + ] }, { "cell_type": "code", diff --git a/trap/base.py b/trap/base.py index df828ed..2b9af00 100644 --- a/trap/base.py +++ b/trap/base.py @@ -145,15 +145,36 @@ class DistortedCamera(ABC): H = self.H coords = cv2.perspectiveTransform(np.array([points]),H) - if coords.shape[1:] == (1,2): - coords = np.reshape(coords, (coords.shape[0], 2)) + # if coords.shape[1:] == (1,2): + coords = np.reshape(coords, (len(points), 2)) return coords + @classmethod + def from_calibfile(cls, calibration_path, H, fps): + with calibration_path.open('r') as fp: + data = json.load(fp) + return cls.from_calibdata(data, H, fps) + + + @classmethod + def from_paths(cls, calibration_path, h_path, fps): + H = H_from_path(h_path) + with calibration_path.open('r') as fp: + calibdata = json.load(fp) + if 'type' in calibdata and calibdata['type'] == 'fisheye': + camera = FisheyeCamera.from_calibdata(calibdata, H, fps) + else: + camera = Camera.from_calibdata(calibdata, H, fps) + return camera + + # return cls.from_calibfile(calibration_path, H, fps) + def points_img_to_world(self, points: PointList, scale = 1.): # undistort & project coords = self.undistort_points(points) - coords: np.ndarray[argparse.Any, np.dtype[np.integer[argparse.Any] | np.floating[argparse.Any]]] = self.project_points(coords, scale) + + coords = self.project_points(coords, scale) return coords class FisheyeCamera(DistortedCamera): @@ -181,6 +202,7 @@ class FisheyeCamera(DistortedCamera): def undistort_points(self, distorted_points: PointList): points = cv2.fisheye.undistortPoints (np.array([distorted_points]).astype(np.float32), K=self.scaled_K, D=self.D, R=self._R, P=self.new_K) + return points[0] @property @@ -191,11 +213,9 @@ class FisheyeCamera(DistortedCamera): def projected_h(self): return self.dim3[1] + @classmethod - def from_calibfile(cls, calibration_path, H, fps): - with calibration_path.open('r') as fp: - data = json.load(fp) - + def from_calibdata(cls, data, H, fps): return cls( data['dim1'], data['dim2'], @@ -207,10 +227,6 @@ class FisheyeCamera(DistortedCamera): data['balance'], H, fps) - @classmethod - def from_paths(cls, calibration_path, h_path, fps): - H = H_from_path(h_path) - return cls.from_calibfile(calibration_path, H, fps) class Camera(DistortedCamera): @@ -225,10 +241,8 @@ class Camera(DistortedCamera): self.newcameramtx, self.roi = cv2.getOptimalNewCameraMatrix(self.mtx, self.dist, (self.w,self.h), 1, (self.w,self.h)) @classmethod - def from_calibfile(cls, calibration_path, H, fps): - with calibration_path.open('r') as fp: - data = json.load(fp) - + def from_calibdata(cls, data, H, fps): + return cls( np.array(data['camera_matrix']), np.array(data['dist_coeff']), @@ -244,10 +258,7 @@ class Camera(DistortedCamera): def projected_h(self): return self.h - @classmethod - def from_paths(cls, calibration_path, h_path, fps): - H = H_from_path(h_path) - return cls.from_calibfile(calibration_path, H, fps) + def undistort_img(self, img: MatLike): return cv2.undistort(img, self.mtx, self.dist, None, self.newcameramtx) @@ -255,7 +266,8 @@ class Camera(DistortedCamera): def undistort_points(self, distorted_points: PointList): points = cv2.undistortPoints(np.array([distorted_points]).astype('float32'), self.mtx, self.dist, None, self.newcameramtx) - return points[0] + # print(points.reshape()) + return points.reshape(points.shape[0], 2) @dataclass diff --git a/trap/cv_renderer.py b/trap/cv_renderer.py index e386235..e52bf10 100644 --- a/trap/cv_renderer.py +++ b/trap/cv_renderer.py @@ -494,8 +494,9 @@ def decorate_frame(frame: Frame, tracker_frame: Frame, prediction_frame: Frame, # cv2.imwrite(str(self.config.output_dir / "orig.png"), warpedFrame) cv2.rectangle(img, (0,0), (img.shape[1],25), (0,0,0), -1) + def conversion(points): - convert_world_points_to_img_points(points, scale) + return convert_world_points_to_img_points(points, scale) if not tracker_frame: cv2.putText(img, f"and track", (650,17), cv2.FONT_HERSHEY_PLAIN, 1, (255,255,0), 1) diff --git a/trap/tracker.py b/trap/tracker.py index 98d62af..b9b292c 100644 --- a/trap/tracker.py +++ b/trap/tracker.py @@ -556,7 +556,7 @@ class Tracker: detections: List[Detection] = self.track_frame(frame) - + # Store detections into tracklets projected_coordinates = [] # now in track_frame() @@ -586,6 +586,7 @@ class Tracker: active_track_ids = [d.track_id for d in detections] active_tracks = {t.track_id: t.get_with_interpolated_history() for t in self.tracks.values() if t.track_id in active_track_ids} active_tracks = displacement_filter.apply_to_dict(active_tracks, frame.camera)# a filter to remove just detecting static objects + # print(len(detections), len(active_tracks)) removable_tracks = [] for track_id, track in self.tracks.items():