alphapose-feedback/find_diffs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "%config Application.log_level='INFO'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import heapq\n",
    "import json\n",
    "from pathlib import Path\n",
    "import pickle\n",
    "import shutil\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "import tqdm\n",
    "from loop_alphapose_training import run_cmd, Iteration\n",
    "import logging\n",
    "import coloredlogs\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycocotools.coco import COCO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = logging.getLogger(__name__)\n",
    "coloredlogs.install(level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_keypoints(ann1, ann2) -> float:\n",
    "    if ann1 is None or ann2 is None:\n",
    "        return None\n",
    "    return np.linalg.norm(np.array(ann1['keypoints']) - np.array(ann2['keypoints']))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bbox_to_xy(bbox) -> list[float, float, float, float]:\n",
    "    bbox_x, bbox_y, bbox_w, bbox_h = bbox\n",
    "    return [bbox_x, bbox_y, bbox_x + bbox_w, bbox_y + bbox_h]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_bbox(bbox1, bbox2) -> float:\n",
    "    return np.linalg.norm(np.array(bbox_to_xy(bbox1))-np.array(bbox_to_xy(bbox2)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from shapely.geometry import box\n",
    "\n",
    "def bbox_iou(bbox1, bbox2) -> float:\n",
    "    \"\"\"intersection over union of 2 bounding boxes\"\"\"\n",
    "    b1 = box(*bbox_to_xy(bbox1))\n",
    "    b2 = box(*bbox_to_xy(bbox2))\n",
    "    return (b1.intersection(b2).area / b1.union(b2).area)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def keypoint_bbox_xy(ann):\n",
    "    # fix too wide bounding boxes by calculating ourselves\n",
    "    kp = np.array(ann['keypoints'])\n",
    "    x = kp[0::3]\n",
    "    y = kp[1::3]\n",
    "    v = kp[2::3]\n",
    "    min_x = min(x)\n",
    "    max_x = max(x)\n",
    "    min_y = min(y)\n",
    "    max_y = max(y)\n",
    "    return [min_x, min_y, max_x, max_y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def boxes_overlap(bbox1_xy, bbox2_xy):\n",
    "    x1min, y1min, x1max, y1max = bbox1_xy\n",
    "    x2min, y2min, x2max, y2max = bbox2_xy\n",
    "    # return x1max >= x2min and x2max >= x1min and y1max >= y2min and y2max >= y1min\n",
    "    return (x1min < x2max and x2min < x1max and y1min < y2max and y2min < y1max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def anns_overlap(ann1, ann2):\n",
    "    return boxes_overlap(keypoint_bbox_xy(ann1), keypoint_bbox_xy(ann2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from shapely.geometry import box\n",
    "\n",
    "def match_annotations(anns1: list[dict], anns2: list[dict]) -> list[tuple]:\n",
    "    # both 1 simple case:\n",
    "    if len(anns1) == 1 and len(anns2) == 1:\n",
    "        if not anns_overlap(anns1[0], anns2[0]):\n",
    "            return [(None, None)]\n",
    "        return [(anns1[0], anns2[0])]\n",
    "    \n",
    "    if len(anns2) == 0:\n",
    "        return [(ann1, None) for ann1 in anns1]\n",
    "    \n",
    "    if len(anns1) == 0:\n",
    "        return [(None, ann2) for ann2 in anns2]\n",
    "\n",
    "    matches = []\n",
    "    # TODO this loop is biased towards the first item in anns1, ideally\n",
    "    # we first calculate the diffs between all annotations, and sort that.\n",
    "    # but this is quicker and perhaps fine for our purpose\n",
    "    \n",
    "    # TODO this propsed solution doesn't work...\n",
    "    # diffs = np.zeros((len(anns1), len(anns2)))\n",
    "    # for i, ann1 in enumerate(anns1):\n",
    "    #     # if len(anns2) == 0: # this can have happended in the loop\n",
    "    #     #     matches.append((ann1, None))\n",
    "    #     # else:\n",
    "    #         diffs[i] = np.array([bbox_iou(ann1['bbox'], ann2['bbox']) for ann2 in anns2])\n",
    "    #         # if ann1['image_id'] in [171729, 269815, 409216]:\n",
    "    #         #     print('diff', ann1['image_id'], diffs, diffs.argmax())\n",
    "    #         # idx = diffs.argmax()\n",
    "\n",
    "    #         # if not anns_overlap(ann1, anns2[idx]):\n",
    "    #         #     # closest match does not overlap. So probably none will\n",
    "    #         #     matches.append((ann1, None))\n",
    "    #         # else:\n",
    "    #         #     match = anns2.pop(idx)\n",
    "    #         #     matches.append((ann1, match))\n",
    "    # anns1_not_found = anns1.copy()\n",
    "    # anns2_not_found = anns2.copy()\n",
    "    # orig_diffs = diffs.copy()\n",
    "    # while len(diffs):\n",
    "    #      idx = np.unravel_index(diffs.argmax(), diffs.shape)\n",
    "    #      a1, a2 = anns1[idx[1]], anns2[idx[0]]\n",
    "    #      diffs = np.delete(np.delete(diffs, idx[1], 1), idx[0], 0)\n",
    "    #      matches.append((a1, a2))\n",
    "    #      anns1_not_found.remove(a1)\n",
    "    #      anns2_not_found.remove(a2)\n",
    "\n",
    "    \n",
    "    # matches.extend([(ann1, None) for ann1 in anns1_not_found])\n",
    "    # matches.extend([(None, ann2) for ann2 in anns2_not_found])\n",
    "    # return matches\n",
    "\n",
    "    # workaround by starting with largest (thus most likely the foremost) first\n",
    "    anns1.sort(key=lambda a: box(*bbox_to_xy(a['bbox'])).area, reverse=True)\n",
    "    for i, ann1 in enumerate(anns1):\n",
    "        if len(anns2) == 0: # this can have happended in the loop\n",
    "            matches.append((ann1, None))\n",
    "        else:\n",
    "            diffs = np.array([bbox_iou(ann1['bbox'], ann2['bbox']) for ann2 in anns2])\n",
    "            if ann1['image_id'] in [171729, 269815, 409216]:\n",
    "                print('diff', ann1['image_id'], diffs, diffs.argmax())\n",
    "            idx = diffs.argmax()\n",
    "\n",
    "            if not anns_overlap(ann1, anns2[idx]):\n",
    "                # closest match does not overlap. So probably none will\n",
    "                matches.append((ann1, None))\n",
    "            else:\n",
    "                match = anns2.pop(idx)\n",
    "                matches.append((ann1, match))\n",
    "    \n",
    "    matches.extend([(None, ann2) for ann2 in anns2])\n",
    "    return matches\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=5.35s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    }
   ],
   "source": [
    "original_path = Path('data/coco/annotations/person_keypoints_train2017.json')\n",
    "\n",
    "coco_orig = COCO(original_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'segmentation': [[154.68,\n",
       "   330.84,\n",
       "   148.23,\n",
       "   327.62,\n",
       "   150.38,\n",
       "   298.62,\n",
       "   150.38,\n",
       "   285.73,\n",
       "   125.68,\n",
       "   247.06,\n",
       "   123.53,\n",
       "   213.76,\n",
       "   117.08,\n",
       "   187.98,\n",
       "   132.12,\n",
       "   164.35,\n",
       "   136.42,\n",
       "   150.38,\n",
       "   135.34,\n",
       "   143.94,\n",
       "   129.97,\n",
       "   138.57,\n",
       "   127.82,\n",
       "   129.97,\n",
       "   128.9,\n",
       "   111.71,\n",
       "   129.97,\n",
       "   108.49,\n",
       "   137.49,\n",
       "   95.6,\n",
       "   138.57,\n",
       "   85.93,\n",
       "   145.01,\n",
       "   71.97,\n",
       "   153.6,\n",
       "   64.45,\n",
       "   185.83,\n",
       "   54.78,\n",
       "   200.87,\n",
       "   87.01,\n",
       "   201.94,\n",
       "   98.82,\n",
       "   200.87,\n",
       "   119.23,\n",
       "   218.05,\n",
       "   146.09,\n",
       "   230.94,\n",
       "   152.53,\n",
       "   238.46,\n",
       "   163.27,\n",
       "   248.13,\n",
       "   180.46,\n",
       "   252.43,\n",
       "   195.5,\n",
       "   259.95,\n",
       "   209.46,\n",
       "   262.09,\n",
       "   230.94,\n",
       "   262.09,\n",
       "   281.43,\n",
       "   262.09,\n",
       "   305.06,\n",
       "   264.24,\n",
       "   334.06,\n",
       "   258.87,\n",
       "   378.1,\n",
       "   258.87,\n",
       "   398.51,\n",
       "   264.24,\n",
       "   438.26,\n",
       "   274.98,\n",
       "   468.33,\n",
       "   257.8,\n",
       "   470.48,\n",
       "   228.8,\n",
       "   451.15,\n",
       "   228.8,\n",
       "   455.44,\n",
       "   229.87,\n",
       "   471.56,\n",
       "   205.16,\n",
       "   472.63,\n",
       "   199.79,\n",
       "   472.63,\n",
       "   183.68,\n",
       "   469.41,\n",
       "   181.53,\n",
       "   468.33,\n",
       "   172.94,\n",
       "   455.44,\n",
       "   168.64,\n",
       "   427.51,\n",
       "   163.27,\n",
       "   363.07]],\n",
       " 'num_keypoints': 15,\n",
       " 'area': 41478.22885,\n",
       " 'iscrowd': 0,\n",
       " 'keypoints': [172,\n",
       "  122,\n",
       "  2,\n",
       "  180,\n",
       "  109,\n",
       "  2,\n",
       "  162,\n",
       "  113,\n",
       "  2,\n",
       "  191,\n",
       "  106,\n",
       "  2,\n",
       "  149,\n",
       "  121,\n",
       "  2,\n",
       "  214,\n",
       "  155,\n",
       "  2,\n",
       "  137,\n",
       "  189,\n",
       "  2,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  157,\n",
       "  276,\n",
       "  2,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  168,\n",
       "  338,\n",
       "  2,\n",
       "  231,\n",
       "  294,\n",
       "  2,\n",
       "  187,\n",
       "  310,\n",
       "  2,\n",
       "  241,\n",
       "  372,\n",
       "  2,\n",
       "  197,\n",
       "  384,\n",
       "  2,\n",
       "  253,\n",
       "  436,\n",
       "  2,\n",
       "  206,\n",
       "  446,\n",
       "  2],\n",
       " 'image_id': 38449,\n",
       " 'bbox': [117.08, 54.78, 157.9, 417.85],\n",
       " 'category_id': 1,\n",
       " 'id': 501841}"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "coco_orig.anns[501841]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_most_different_annotations(coco_orig: COCO, coco_iter: COCO, n: int = 10, only_full=False, only_singular=False) -> tuple[int, float, dict, dict]:\n",
    "    matched_annotations = []\n",
    "    skipped_many = 0\n",
    "    for img_id, img in tqdm.tqdm(coco_orig.imgs.items()):\n",
    "        # loop over images and get all annotations for each\n",
    "        anns_iter = [coco_iter.anns[ann_id] for ann_id in coco_iter.getAnnIds(imgIds = img_id)]\n",
    "        anns_orig = [coco_orig.anns[ann_id] for ann_id in coco_orig.getAnnIds(imgIds = img_id)]\n",
    "        \n",
    "        # optimise for now: filter images with just one figure\n",
    "        if only_singular and (len(anns_iter) > 1 or len(anns_orig) > 1):\n",
    "            skipped_many+=1\n",
    "            continue\n",
    "        if only_full:\n",
    "            anns_iter = [ann for ann in anns_iter if all(ann['keypoints'])]\n",
    "            anns_orig = [ann for ann in anns_orig if all(ann['keypoints'])]\n",
    "        matched_annotations.extend([(img_id, ann1, ann2) for ann1, ann2 in match_annotations(anns_orig, anns_iter)])\n",
    "        \n",
    "    match_losses = []\n",
    "    skipped_empty, skipped_kp = 0, 0\n",
    "    for img_id, ann1, ann2 in tqdm.tqdm(matched_annotations):\n",
    "        if ann1 is None or ann2 is None:\n",
    "            # todo: we later want to store these somewhere else\n",
    "            logger.debug(f'skipping hallucination or removal in {img_id}')\n",
    "            skipped_empty+=1\n",
    "            continue\n",
    "        if ann1['num_keypoints'] < 3:\n",
    "            logger.debug(f'skipping small number of keypoints in {img_id}')\n",
    "            skipped_kp+=1\n",
    "            continue\n",
    "        match_losses.append((img_id, diff_keypoints(ann1, ann2), ann1, ann2))\n",
    "\n",
    "    logger.info(f'Skipped {skipped_many} images with > 1 images')\n",
    "    logger.info(f'Skipped {skipped_empty} hallucinations or removals')\n",
    "    logger.info(f'Skipped {skipped_kp} annotations with small nr of keypoints')\n",
    "\n",
    "    logger.info(f'Found {len(match_losses)} annotation pairs')\n",
    "\n",
    "    return heapq.nlargest(n, match_losses, key=lambda ml: ml[1])\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](coco-keypoints.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[15, 13],\n",
       "       [13, 11],\n",
       "       [16, 14],\n",
       "       [14, 12],\n",
       "       [11, 12],\n",
       "       [ 5, 11],\n",
       "       [ 6, 12],\n",
       "       [ 5,  6],\n",
       "       [ 5,  7],\n",
       "       [ 6,  8],\n",
       "       [ 7,  9],\n",
       "       [ 8, 10],\n",
       "       [ 1,  2],\n",
       "       [ 0,  1],\n",
       "       [ 0,  2],\n",
       "       [ 1,  3],\n",
       "       [ 2,  4],\n",
       "       [ 3,  5],\n",
       "       [ 4,  6]])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(coco_orig.loadCats(1)[0]['skeleton'])-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.collections import PatchCollection\n",
    "from matplotlib.patches import Polygon\n",
    "\n",
    "def plotKeypointAnns(coco, anns: list, colours: list, draw_bbox = False, colour_skeleton=True):\n",
    "    \"\"\"\n",
    "    Modifiied from pycocotools to draw annotations in predefined colours\n",
    "    \"\"\"\n",
    "    if len(anns) == 0:\n",
    "        return 0\n",
    "\n",
    "    ax = plt.gca()\n",
    "    ax.set_autoscale_on(False)\n",
    "    polygons = []\n",
    "    color = []\n",
    "    for i, ann in enumerate(anns):\n",
    "        c = colours[i % len(colours)]\n",
    "        if 'keypoints' in ann and type(ann['keypoints']) == list:\n",
    "            # turn skeleton into zero-based index\n",
    "            sks = np.array(coco.loadCats(ann['category_id'])[0]['skeleton'])-1\n",
    "            kp = np.array(ann['keypoints'])\n",
    "            x = kp[0::3]\n",
    "            y = kp[1::3]\n",
    "            v = kp[2::3]\n",
    "            for sk in sks:\n",
    "                if np.all(v[sk]>0):\n",
    "                    if colour_skeleton:\n",
    "                        if sk[1] <= 4:\n",
    "                            c = (0,1,0)\n",
    "                        elif all(sk == [3,5]) or all(sk == [4,6]):\n",
    "                            # skip, eye-shoulder\n",
    "                            continue\n",
    "                        elif sk[1] >= 11 and sk[0] > 12:\n",
    "                            c = (1,0,0)\n",
    "                        elif sk[1] in [12, 11,  6]:\n",
    "                            c = (1,1,0)\n",
    "                        elif sk[1] >= 7:\n",
    "                            c = (0,1,1)\n",
    "                        else:\n",
    "                            c = (1,0,1)\n",
    "                            \n",
    "                    plt.plot(x[sk],y[sk], linewidth=3, color=c)\n",
    "            # plt.plot(x[v>0], y[v>0],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2)\n",
    "            # plt.plot(x[v>1], y[v>1],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2)\n",
    "\n",
    "        if draw_bbox:\n",
    "            [bbox_x, bbox_y, bbox_w, bbox_h] = ann['bbox']\n",
    "            poly = [[bbox_x, bbox_y], [bbox_x, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y]]\n",
    "            np_poly = np.array(poly).reshape((4,2))\n",
    "            polygons.append(Polygon(np_poly))\n",
    "            color.append(c)\n",
    "\n",
    "    p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4)\n",
    "    ax.add_collection(p)\n",
    "    p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2)\n",
    "    ax.add_collection(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "# annIds = coco_iter1.getAnnIds(imgIds=results[0][0], iscrowd=None)\n",
    "# fn = Path('data/coco/train2017/') / coco_iter1.imgs[results[0][0]]['file_name']\n",
    "# plt.imshow(Image.open(fn)); plt.axis('off')\n",
    "# print(annIds)\n",
    "# print(results[0][2])\n",
    "# anns = coco_iter1.loadAnns(annIds)\n",
    "# print(anns)\n",
    "# coco_iter1.showAnns([results[0][2],results[0][3]])\n",
    "# plt.show()\n",
    "# plt.savefig()\n",
    "\n",
    "def copy_annotation_matches_to_path(coco_src: COCO, path: Path, results: list[int, float, dict, dict], visualise: bool = True):\n",
    "    # display(Image.open(largest_diff_files[0]))\n",
    "    for img_id, diff, ann1, ann2 in results:\n",
    "        source = Path('data/coco/train2017/') / coco_src.imgs[img_id]['file_name']\n",
    "        fn = source.name\n",
    "        dest = path / fn\n",
    "        if not visualise:\n",
    "            logger.info(f\"Copy {source} -> {dest}\")\n",
    "            shutil.copy(source, dest)\n",
    "        else:\n",
    "            logger.info(f\"Plot {source} -> {dest}\")\n",
    "            im = Image.open(source)\n",
    "            plt.imshow(im)\n",
    "            # plt.axis('off')\n",
    "            plotKeypointAnns(coco_orig, [ann1, ann2], [(0.7, 0.2, 0.0), (0.0, 0.2, 0.7)], True)\n",
    "            plt.show()\n",
    "            plt.savefig(dest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# i=10\n",
    "# iteration = Iteration(i)\n",
    "# if not iteration.merged_results_path('train').exists():\n",
    "#     print('not existing')\n",
    "#     # break\n",
    "# else:\n",
    "#     print(i)\n",
    "#     iteration_path = iteration.merged_results_path('train')\n",
    "#     coco_iterI = COCO(iteration_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # results = find_most_different_annotations(coco_orig, coco_iter, 10)\n",
    "# results = find_most_different_annotations(coco_orig, coco_iterI, 10, True, False)\n",
    "# out_path = Path(f'./data/test_imgs0-{i}/')\n",
    "# if not out_path.exists():\n",
    "#     out_path.mkdir()\n",
    "# # copy_annotation_matches_to_path(coco_orig, out_path, results)\n",
    "# # break # only one for now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(coco_orig.getAnnIds(imgIds = 38449),coco_iterI.getAnnIds(imgIds = 38449))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [ml[1] for ml in results]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[501841, 507179]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coco_orig.getAnnIds(imgIds=38449)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(results[0][0])\n",
    "# print(results[0][2]['bbox'], results[0][3]['bbox'])\n",
    "# print(anns_overlap(results[0][2], results[0][3]))\n",
    "# print('kp box 1', keypoint_bbox_xy(results[0][2]))\n",
    "# print('kp box 2', keypoint_bbox_xy(results[0][3]))\n",
    "# # print(bbox_to_xy(results[0][2]['bbox']))\n",
    "\n",
    "# results[0][2]['keypoints'], results[0][3]['keypoints']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# copy_annotation_matches_to_path(coco_orig, out_path, results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load from Pickle out/interesting_results.pcl\n"
     ]
    }
   ],
   "source": [
    "# TODO: \n",
    "# coco_orig=None\n",
    "# compare with first run, so that we always have same type of annotation (coco can be slightly different)\n",
    "# coco_iter1 = COCO(Iteration(1).merged_results_path('train'))\n",
    "\n",
    "results_path = Path('out/interesting_results.pcl')\n",
    "if  results_path.exists():\n",
    "    with results_path.open('rb') as fp:\n",
    "        print(f\"load from Pickle {results_path}\")\n",
    "        all_results = pickle.load(fp)\n",
    "else:\n",
    "    \n",
    "    i = 1\n",
    "    all_results=[]\n",
    "    while True:\n",
    "        i+=1\n",
    "        iteration = Iteration(i)\n",
    "        if not iteration.merged_results_path('train').exists():\n",
    "            break\n",
    "        print(i)\n",
    "        iteration_path = iteration.merged_results_path('train')\n",
    "        coco_iterI = COCO(iteration_path)\n",
    "        # results = find_most_different_annotations(coco_orig, coco_iter, 10)\n",
    "        results = find_most_different_annotations(coco_orig, coco_iterI, 30, True, False)\n",
    "        all_results.extend(results)\n",
    "        # out_path = Path(f'./data/test_imgs0-{i}/')\n",
    "        # if not out_path.exists():\n",
    "        #     out_path.mkdir()\n",
    "        # copy_annotation_matches_to_path(coco_orig, out_path, results)\n",
    "        # break # only one for now\n",
    "    with results_path.open('wb') as fp:\n",
    "        pickle.dump(all_results, fp)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "interesting_img_ids = set([r[0] for r in all_results])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(interesting_img_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coco_i(n) -> COCO:\n",
    "    if n == 0:\n",
    "        return coco_orig\n",
    "    \n",
    "    iteration = Iteration(n)\n",
    "    if not iteration.merged_results_path('train').exists():\n",
    "        raise RuntimeError(\"Iteration not found\")\n",
    "    \n",
    "    iteration_path = iteration.merged_results_path('train')\n",
    "    return COCO(iteration_path)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_image_for_coco_i(i, coco, img_id, out_path):\n",
    "    # display(Image.open(largest_diff_files[0]))\n",
    "    anns = [coco.anns[ann] for ann in coco.getAnnIds(imgIds = img_id)]\n",
    "    \n",
    "    # for img_id, diff, ann1, ann2 in results:\n",
    "    source = Path('data/coco/train2017/') / coco.imgs[img_id]['file_name']\n",
    "    fn = source.name\n",
    "    dest = out_path / fn\n",
    "\n",
    "    dest = dest.with_name(f'{source.stem}-{i:04}{source.suffix}')\n",
    "    # if not visualise:\n",
    "    #     logger.info(f\"Copy {source} -> {dest}\")\n",
    "    #     shutil.copy(source, dest)\n",
    "    # else:\n",
    "    logger.debug(f\"Plot {source} -> {dest}\")\n",
    "    im = Image.open(source)\n",
    "    plt.clf() # reset residue figure when in loop\n",
    "    plt.imshow(im)\n",
    "    plt.axis('off')\n",
    "    plotKeypointAnns(coco_orig, anns, [(1,1,1)], False)\n",
    "    # plt.show()\n",
    "    plt.savefig(dest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_images_for_coco_i(i, coco, img_ids, out_path):\n",
    "    for img_id in tqdm.tqdm(img_ids):\n",
    "        draw_image_for_coco_i(i, coco, img_id, out_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_path = Path(f'./out/interesting_imgs/')\n",
    "if not out_path.exists():\n",
    "    out_path.mkdir()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 16.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "loading annotations into memory...\n",
      "Done (t=9.93s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:07<00:00, 13.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "loading annotations into memory...\n",
      "Done (t=8.61s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:09<00:00, 10.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "loading annotations into memory...\n",
      "Done (t=5.73s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:08<00:00, 11.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "loading annotations into memory...\n",
      "Done (t=7.96s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 14.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "loading annotations into memory...\n",
      "Done (t=8.06s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:08<00:00, 11.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "loading annotations into memory...\n",
      "Done (t=8.02s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 14.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "loading annotations into memory...\n",
      "Done (t=8.31s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 14.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "loading annotations into memory...\n",
      "Done (t=8.09s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:08<00:00, 11.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "loading annotations into memory...\n",
      "Done (t=8.16s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 14.57it/s]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQoAAAGFCAYAAAAFLb3EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOz9aaxu2Xnnh/3WsId3OtOdh7o1sqpINkWKFEWp2ZZa7Z7kdltAe0ocJDGSRgIjQIJ8ChADQT7kW2LE8JcMNpykPcGJh3Y63W7LPVAtqTVSFEWyyJrr1p3vmd9xD2utJx+etd9zixp4BRBIPtQGi6fq3nPes/deaz3D//k//8eIiPDp9en16fXp9Sdc9v/XN/Dp9en16fX//9enhuLT69Pr0+tHXp8aik+vT69Prx95fWooPr0+vT69fuT1qaH49Pr0+vT6kdenhuLT69Pr0+tHXp8aik+vT69Prx95fWooPr0+vT69fuTln/cb/zd/8y/jS4cxIFiMMYDBGIO1BmMcINs/BzDGYBAEg8GSUiIhGGsxtsA6D8aSKInGs2iEvk8YK1gMBuj7jp2dffoofPzgY95+5/uMR3vM56ckMVRlRUgRSBhgd3fMP/dXfpHd/R2ctdRlSVEUOAcpRUQSVVUiApvNhtV6Q1kWjKqKsqoQSTRNS+gjVV1QFCXWWsDQ9T0h6NOlmFhv1qw3a07OTjg6fMLx8VMePX7A0dEJy+UaK8JsPOLqlV1efOE6d+68xOc//zNMd6/w0UcfUZUj9vanVKPEerOkaVuaNuComIx3qcsKV1qMMyAG5yKuLilthdiS1bxhtVrRNBtO5qc8evgB737wLu+9e5cHD5/QdQmL1fu3DgzMCsfYW1abBlt4knMYwDuDtxZrDM5anPcUtsA7cMYgxtF2LV3bAIl6OqLrIYjDeU9VVfiioqorxqMJZVnQtD19jIhEMJbDJ485fvAxMSSw4KwlCBhJlA7EOIqqZt32WFsg1pAAEgiJlCIpCaSkXwERQUT3m35JkAw7Fbx8UBKSoUmBv/zl2/zCz/1l/p3/6D+hCpY710r+zX/vP+Pg6pcxw89t93W+BJIIxiQQQyTwd//Tf5t3v/ENNjFxct7z1rsfcHz4hH/zX/0Sf/2f+Rnmyw4Ry9/95vv8b//DX6a0NYZAlIQkQ4vgcZhC10S8oayg8AFrLH0HIQjeO6raU9YFbbui8ELXB0geayOpS0RxdCGQose6nrKyiDj6LuILS4qBGEAkksRgZHi+hIieV2vg3fvtj89QGKub1eRDszUExiAMRsMiBgwWti88/62xWGNBBKzLhsUDFmscYiyGhEG/Dpe1FmMt0vekGDEYnHMkifS94KwloRsJEYw1OGcpnMU7h7MGaxLWWkQEY/S+RZIaF2sw2eCBbj5nHVIIxjokQdt2tF3HfL1ivV5xenrE4dEjHj54yNHRMfPzc0IXqMuSSwczvvjZl3j5xTtcv/YCBwfXmUwvMx6NKauaxfqETXfMdNeSYsdic86qdRR+ynR8wN6OwXlDWXmcczhfUVYjvCvwxQRsxWa14vHxQ9555w/4wfe/y/0PP+aj+484PZ7TByEJWOf12Z3BOoczBiuJysFsMqaNEWsdYtDf4yzeWgpf4Iy+Q1c4nHMUZU1RT3j85BHz1YrCGab7+0z39jGuBGNw3uc1twQsXdOxWTdsmg1919CHyHp+jiR1HH0CKxrQOt1MDD4mxkTftUSj3ytJP1okZaMgGDFcUIov9os6q4S3+nl9NMRgGVvPS9d3+frnXuf3v/M9ptMpdTVFCBg8iB229TOfJHkbG72vPrI5n2ONA0mICF3osQZm07EaA4QUAyfzJTEKeMlGzSBG73t75xYsBglCL1B4g5BwTp1qs+kw1oMUrNctiMMYoSog1Q4bLVE8oY9YD0WpzjX0hr4NYBJQgAhuMKbDL87nkufkZf8pDIUDY3WhtkbAZituMcZuF3uIKvRbnf68sbrgCNZlQ2EcKf+sMU4PuhmMioAI1mrEIiQg6vJJIqaY70e2CywCVVVRFgXeW5w1+tV5nLUoWV0NWkoRweK9x7kCwZKSIUah6RKr5ZqmOWSzWXByfszjJ0948vSQo8NjNssVRiL7sykv3LjGC1/+M9y+9QLXrt5kb/86o9EOYgzYSAhr1u2c82ZOJSOWmx5Jnrq+nI2gwSIk0zEaGSbTHWZ7V6jqKZIsTdMRJXE+P+Ho6B3e//A9vvedP+Ct73/I2ZOnrBYtKVlM4TDOUXlHHwIiYIyQJGJSwjqHNUKfLOfrDc4XWGvwNhsD7/S/vaesK6rxCD+aURQVRTHCFRWHZytCekzpHYtVQysbfBlBhJgifYyEPiICMfT0fU8IPSklYkzQtzhjCHkd1HAbMIIzgDc5YhWi5AOWjU9KktdPN3je5jle1d1uMOqHjNOcWsCYhDWRB4cnFLLmF3/hZ5lNDS/fuUFZXULfvv4a88OW4plLxBBDT7dcqAOhJ4oQQo93hlldop46ISlxPF9hxOra5s9IIlgxiLMahXsgiRoeLCHEfD/6hCEkFucLRuOSGLKjNoG+E2xh8IWhxOaoB6zr8R586wkiOO/1e62oLc0RRQ7GENB9+hzXcxsKMRrcG7HbqMLYC8Ognhmsddvvt8Zmb23z4QdsNjRiwVis8RoWZyvHYMWFbYRgrC5mTAmsIYrQh4CzVb67i+ilLgvKwmtU4S1FaXBO/85iENFkKIklZWvcbBpi7FmvTzk9P+Lw8CmPHj7g6PCI1WJJ27bMZhNuXb/CP/NTb/LC7Ttcu/oiu7uXqOoZrhgjJpKkoQ9rTlf3Wa6W9EGwpsC7KVU9wfmSsgpYo2G0sZHRpKaux3hfMZtdohxNEREWizmHTx/ygx98m/fef5u333mbjz54xMlpS9/3eON58+YOqRrz8ekpVelpsudyxiKSsMZifUFhwTmwzmBQg195h81RhBiDGxWMJhOqekYxmlJUIxBD2wWWq5Y+rDhfLLC5NWizbpgvn2DLUjdfSoQY9EBH0VQhCilHbpLAxYAfooRnWoysAWfV4IvR9HS77xBNXcRsTYNeEUGdiR50Q0q6d5JEvLMYBGcsReE4PF2znp/xyquf4erOn2d6cBnrRltDMfyuP9pYaLjedw3dakEEUoKQEikm6sJwMBvrvSSNaE7XGz0ExmbP3SM4vBMme/tsuiVF7Wg2PVEsVtQYqTVU0yIC1kIIHWCJMeGdIQZDFPClUBSOsio4PZ1TlpbJzNBsAjFYMPrurMtvLQkp730EnIB5TpTyT2EohqjBbg++WgmNCw1ma/3VkICxFovJOf7FZ2jEodEExoG1uOTAgsSLxRkWzhhLTEKMYG2Bde7ipQ7RhP5CjLU473AOnCswziI4YoS2i/RdR9OuOT8/5/TslCdPH3F8/IjjwyNOT8/ou47xaMS1awd86QtvcOeF21y/8gJ7e9epp7uUxRhjIMqapl0wb56yPlnTND0ilqKYUFcjJqNbuDIbshQIqaUPAe883sHewSXKyZS6mpCkoGnWPHhyn8OjB3z4wfu8++67vPPuBzx+eMxq2ZJEUwcosBYkRqwRZpMSc2pBHKRIEg1dfeEwNmEtFLbAkHKK4fA4xOhx7KJQjmqk3iGWY1YY+sWCcHJK17Q06w2hD0iM9MszXEoIeij7tiV0/XaPpJQQQT1WDs3JEYIRcNnwS15ZjP67/qvgnCHwTAawjRSH+CGnJ5JDeRmi1xxRGLP9MWut/k4TqbzQJc+9owe89Jk32ZlOqfZ3MH5IYxhCik9eah8YYvS+a+jbNfoDlpASkoTpyDGbjCDp86UonKxWYDUtT3pziGha+fKrr/Lx43uEdokvNJ0KIWBkSHeSRuBqBxERUoqaooviJX2fSEBVeeraURQOEaEqHUUBm7UQ2zQAN3pfYkgpogdNiBev7kdez596GJ8xiXzYt1iF0cjBeT3UGRATNKLAAsYpgAkYp7mgSRaLJ7pscHohmYh9JhTUVEQfKsVAihFJCnBKUmuYALL3FCOsNg19hBCFruvpz9asVwvOFqccnx1zdHjEkydPOT06om86fOHZPxj
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "for i in range(10):\n",
    "    print(i)\n",
    "    coco = get_coco_i(i)\n",
    "    draw_images_for_coco_i(i, coco, interesting_img_ids, out_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "video_out_path = Path(f'./out/alphapose-feedback-videos/')\n",
    "if not video_out_path.exists():\n",
    "    video_out_path.mkdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import subprocess\n",
    "\n",
    "\n",
    "images = {}\n",
    "for p in out_path.iterdir():\n",
    "    if p.suffix != '.jpg':\n",
    "        continue\n",
    "\n",
    "    img_id, nr = p.stem.split('-')\n",
    "    if img_id not in images:\n",
    "        images[img_id] = []\n",
    "    \n",
    "    images[img_id].append(nr)\n",
    "\n",
    "for nrs in images.values():\n",
    "    nrs.sort()\n",
    "\n",
    "for img_id, imgs in tqdm.tqdm(images.items()):\n",
    "    pattern = str(out_path) + f'/{img_id}-%04d.jpg'\n",
    "    out_file = str(video_out_path) + f'/{img_id}.mp4'\n",
    "    # cmd=[\n",
    "    #     'ffmpeg',\n",
    "    #     '-y',\n",
    "    #     '-r', '2',\n",
    "    #     '-start_number', '0',\n",
    "    #     '-i', pattern,\n",
    "    #     '-c:v', 'libx264',\n",
    "    #     # '-vf', 'fps=25',\n",
    "    #     # '-vf', \"drawtext=fontfile=Arial.ttf: text='%{frame_num}': start_number=0: x=(w-tw)/2: y=h-(2*lh): fontcolor=black: fontsize=20: box=1: boxcolor=white: boxborderw=5\",\n",
    "    #     # '-pix_fmt', 'yuv420p',\n",
    "    #     out_file\n",
    "    # ]\n",
    "    ! ffmpeg -y -r 2 -start_number 0 -i {pattern} -c:v libx264 -vf fps=25 -vf \"drawtext=fontfile=Arial.ttf: text='%{{frame_num}}': start_number=0: x=(w-tw)/2: y=h-(2*lh): fontcolor=black: fontsize=20: box=1: boxcolor=white: boxborderw=5\" -pix_fmt yuv420p {out_file}\n",
    "    # break\n",
    "    # ! ls {out_path}\n",
    "    # print(\" \".join(cmd))\n",
    "    # process = subprocess.run(cmd, timeout=30)\n",
    "    # print(process.pid)\n",
    "    # cmd = ['echo', str(out_path)]\n",
    "    # process = subprocess.call(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, shell=True)\n",
    "    \n",
    "    # print(process.pid)\n",
    "    # stdout, stderr = process.communicate()\n",
    "    # print( \"show something after subprocess\")\n",
    "\n",
    "    # print(\" \".join(cmd))\n",
    "    # subprocess.run(cmd)\n",
    "    # break\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17195"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "video_html = \"\"\n",
    "for img_id in images:\n",
    "    video_html += f\"\"\"\n",
    "    <div class='interesting_video'>\n",
    "        <video src=\"{img_id}.mp4\" autoplay loop muted></video>\n",
    "        <span class='id'>{img_id}</span>\n",
    "    </div>\n",
    "    \"\"\"\n",
    "\n",
    "page = f\"\"\"\n",
    "<html>\n",
    "<title>Alphapose feedback on coco</title>\n",
    "<style>\n",
    "body{{font-family: sans-serif;}}\n",
    ".controls{{position:sticky; top:10px;}}\n",
    ".interesting_video{{\n",
    "    display: inline-block;\n",
    "    width: 640px;\n",
    "    text-align: center;\n",
    "}}\n",
    "</style>\n",
    "<body>\n",
    "<h1>Alphapose feedback on coco</h1>\n",
    "<div class='controls'>\n",
    "<input type='range' min=0 max=9 id='time'>\n",
    "</div>\n",
    "<div id=\"video_grid\">\n",
    "{video_html}\n",
    "</div>\n",
    "\n",
    "<script>\n",
    "const videos = document.getElementsByTagName('video');\n",
    "const control = document.getElementById('time');\n",
    "control.addEventListener('mousedown', function(){{\n",
    "    for (v of videos){{ v.pause() }}\n",
    "}})\n",
    "control.addEventListener('mouseup', function(){{\n",
    "    for (v of videos){{ v.play() }}\n",
    "}})\n",
    "control.addEventListener('input', function(e){{\n",
    "    for (v of videos){{ v.currentTime = e.target.value/2; }}\n",
    "}})\n",
    "</script>\n",
    "</body>\n",
    "</html>\n",
    "\"\"\"\n",
    "\n",
    "out_file = video_out_path / \"index.html\"\n",
    "\n",
    "out_file.write_text(page)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "58d01810a2ada16aebdd7a83b5fa4c0faa8a7d1bffdb0f6369e5c75cce3e288d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}