{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vehicles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FDE Results for: int_ee\n",
      "-----------------PH: 2 -------------------\n",
      "FDE Mean @1.0s: 0.15211091398370916\n",
      "RB Viols @1.0s: 0.0024947383125349227\n",
      "FDE @1.0s: 0.06731242196606527\n",
      "KDE @1.0s: -4.282745726122304\n",
      "----------------------------------------------\n",
      "-----------------PH: 4 -------------------\n",
      "FDE Mean @2.0s: 0.6665500898672814\n",
      "RB Viols @2.0s: 0.006711305643509033\n",
      "FDE @2.0s: 0.4448624482250238\n",
      "KDE @2.0s: -2.817182897766484\n",
      "----------------------------------------------\n",
      "-----------------PH: 6 -------------------\n",
      "FDE Mean @3.0s: 1.6002716865301074\n",
      "RB Viols @3.0s: 0.03183521139877072\n",
      "FDE @3.0s: 1.1269273173570473\n",
      "KDE @3.0s: -1.672070802164807\n",
      "----------------------------------------------\n",
      "-----------------PH: 8 -------------------\n",
      "FDE Mean @4.0s: 2.9889876939803504\n",
      "RB Viols @4.0s: 0.08143642515676414\n",
      "FDE @4.0s: 2.1726876209006143\n",
      "KDE @4.0s: -0.7623941477599536\n",
      "----------------------------------------------\n",
      "\n",
      "FDE Results for: int_ee_me\n",
      "-----------------PH: 2 -------------------\n",
      "FDE Mean @1.0s: 0.17146557916606217\n",
      "RB Viols @1.0s: 0.002893866020984665\n",
      "FDE @1.0s: 0.06825468723974978\n",
      "KDE @1.0s: -4.174627329473856\n",
      "----------------------------------------------\n",
      "-----------------PH: 4 -------------------\n",
      "FDE Mean @2.0s: 0.6874934647937203\n",
      "RB Viols @2.0s: 0.006347814614763767\n",
      "FDE @2.0s: 0.4473287549407142\n",
      "KDE @2.0s: -2.7424043655184898\n",
      "----------------------------------------------\n",
      "-----------------PH: 6 -------------------\n",
      "FDE Mean @3.0s: 1.6150508554078604\n",
      "RB Viols @3.0s: 0.027944558266592166\n",
      "FDE @3.0s: 1.1370181075818808\n",
      "KDE @3.0s: -1.616241617749356\n",
      "----------------------------------------------\n",
      "-----------------PH: 8 -------------------\n",
      "FDE Mean @4.0s: 2.9834139311814645\n",
      "RB Viols @4.0s: 0.07611557086980816\n",
      "FDE @4.0s: 2.2067347028461923\n",
      "KDE @4.0s: -0.7050671606779637\n",
      "----------------------------------------------\n",
      "\n",
      "FDE Results for: vel_ee_me\n",
      "-----------------PH: 2 -------------------\n",
      "FDE Mean @1.0s: 0.21398885662219846\n",
      "RB Viols @1.0s: 0.0024283075681380767\n",
      "FDE @1.0s: 0.1792272294232774\n",
      "KDE @1.0s: 0.8111385940397233\n",
      "----------------------------------------------\n",
      "-----------------PH: 4 -------------------\n",
      "FDE Mean @2.0s: 0.715463329547642\n",
      "RB Viols @2.0s: 0.006407897187558204\n",
      "FDE @2.0s: 0.5706283482566946\n",
      "KDE @2.0s: 0.051893685490453464\n",
      "----------------------------------------------\n",
      "-----------------PH: 6 -------------------\n",
      "FDE Mean @3.0s: 1.5440473025828012\n",
      "RB Viols @3.0s: 0.02805111131806047\n",
      "FDE @3.0s: 1.2515989489585615\n",
      "KDE @3.0s: 0.371638561867866\n",
      "----------------------------------------------\n",
      "-----------------PH: 8 -------------------\n",
      "FDE Mean @4.0s: 2.714255228812044\n",
      "RB Viols @4.0s: 0.06920216365555348\n",
      "FDE @4.0s: 2.2400267464847876\n",
      "KDE @4.0s: 0.8726346089263975\n",
      "----------------------------------------------\n",
      "\n",
      "FDE Results for: robot\n",
      "-----------------PH: 2 -------------------\n",
      "FDE Mean @1.0s: 0.1295215269389519\n",
      "RB Viols @1.0s: 0.0026757717999638924\n",
      "FDE @1.0s: 0.07820393052295552\n",
      "KDE @1.0s: -3.906838146881899\n",
      "----------------------------------------------\n",
      "-----------------PH: 4 -------------------\n",
      "FDE Mean @2.0s: 0.45962341869964574\n",
      "RB Viols @2.0s: 0.0053363964614551365\n",
      "FDE @2.0s: 0.3403511030418785\n",
      "KDE @2.0s: -2.7593676749477294\n",
      "----------------------------------------------\n",
      "-----------------PH: 6 -------------------\n",
      "FDE Mean @3.0s: 1.02267032097404\n",
      "RB Viols @3.0s: 0.016484509839321176\n",
      "FDE @3.0s: 0.805915047871091\n",
      "KDE @3.0s: -1.7502450775203158\n",
      "----------------------------------------------\n",
      "-----------------PH: 8 -------------------\n",
      "FDE Mean @4.0s: 1.8380306576706953\n",
      "RB Viols @4.0s: 0.042144791478606246\n",
      "FDE @4.0s: 1.4979755853506684\n",
      "KDE @4.0s: -0.9291549495198915\n",
      "----------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model in ['int_ee', 'int_ee_me', 'vel_ee', 'robot']:\n",
    "    print(f\"FDE Results for: {model}\")\n",
    "    for ph in [2, 4, 6, 8]:\n",
    "        print(f\"-----------------PH: {ph} -------------------\")\n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}_{ph}_fde_full.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        \n",
    "        print(f\"FDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}\")\n",
    "        del perf_df      \n",
    "        \n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}_{ph}_rv_full.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"RB Viols @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].sum() / (len(perf_df['value'][perf_df['type'] == 'full'].index)*2000)}\")\n",
    "        del perf_df\n",
    "\n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}*_{ph}_fde_most_likely_z.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"FDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'ml'].mean()}\")      \n",
    "        del perf_df\n",
    "              \n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}*_{ph}_kde_full.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"KDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}\")      \n",
    "        print(\"----------------------------------------------\")\n",
    "        del perf_df\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FDE Results for: int_ee_me_no_ego\n",
      "-----------------PH: 2 -------------------\n",
      "FDE Mean @1.0s: 0.16815279412540554\n",
      "RB Viols @1.0s: 0.002895014589929844\n",
      "FDE @1.0s: 0.06937045846177256\n",
      "KDE @1.0s: -4.262019931215572\n",
      "----------------------------------------------\n",
      "-----------------PH: 4 -------------------\n",
      "FDE Mean @2.0s: 0.6655379721067188\n",
      "RB Viols @2.0s: 0.006153364996585336\n",
      "FDE @2.0s: 0.4359008486971371\n",
      "KDE @2.0s: -2.856656149202157\n",
      "----------------------------------------------\n",
      "-----------------PH: 6 -------------------\n",
      "FDE Mean @3.0s: 1.546091556287448\n",
      "RB Viols @3.0s: 0.027780530204259017\n",
      "FDE @3.0s: 1.0896218245514429\n",
      "KDE @3.0s: -1.7563896369106704\n",
      "----------------------------------------------\n",
      "-----------------PH: 8 -------------------\n",
      "FDE Mean @4.0s: 2.8358865412257397\n",
      "RB Viols @4.0s: 0.07581256596510834\n",
      "FDE @4.0s: 2.0939721352439022\n",
      "KDE @4.0s: -0.8690706892091696\n",
      "----------------------------------------------\n",
      "\n",
      "FDE Results for: robot\n",
      "-----------------PH: 2 -------------------\n",
      "FDE Mean @1.0s: 0.1295215269389519\n",
      "RB Viols @1.0s: 0.0026757717999638924\n",
      "FDE @1.0s: 0.07820393052295552\n",
      "KDE @1.0s: -3.906838146881899\n",
      "----------------------------------------------\n",
      "-----------------PH: 4 -------------------\n",
      "FDE Mean @2.0s: 0.45962341869964574\n",
      "RB Viols @2.0s: 0.0053363964614551365\n",
      "FDE @2.0s: 0.3403511030418785\n",
      "KDE @2.0s: -2.7593676749477294\n",
      "----------------------------------------------\n",
      "-----------------PH: 6 -------------------\n",
      "FDE Mean @3.0s: 1.02267032097404\n",
      "RB Viols @3.0s: 0.016484509839321176\n",
      "FDE @3.0s: 0.805915047871091\n",
      "KDE @3.0s: -1.7502450775203158\n",
      "----------------------------------------------\n",
      "-----------------PH: 8 -------------------\n",
      "FDE Mean @4.0s: 1.8380306576706953\n",
      "RB Viols @4.0s: 0.042144791478606246\n",
      "FDE @4.0s: 1.4979755853506684\n",
      "KDE @4.0s: -0.9291549495198915\n",
      "----------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model in ['int_ee_me_no_ego', 'robot']:\n",
    "    print(f\"FDE Results for: {model}\")\n",
    "    for ph in [2, 4, 6, 8]:\n",
    "        print(f\"-----------------PH: {ph} -------------------\")\n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}_{ph}_fde_full.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        \n",
    "        print(f\"FDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}\")\n",
    "        del perf_df      \n",
    "        \n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}_{ph}_rv_full.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"RB Viols @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].sum() / (len(perf_df['value'][perf_df['type'] == 'full'].index)*2000)}\")\n",
    "        del perf_df\n",
    "\n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}*_{ph}_fde_most_likely_z.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"FDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'ml'].mean()}\")      \n",
    "        del perf_df\n",
    "              \n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}*_{ph}_kde_full.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"KDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}\")      \n",
    "        print(\"----------------------------------------------\")\n",
    "        del perf_df\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pedestrians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FDE Results for: int_ee_me_ped\n",
      "-----------------PH: 2 -------------------\n",
      "FDE Mean @1.0s: 0.03182535279935429\n",
      "ADE Mean @1.0s: 0.034975306849922005\n",
      "KDE Mean @1.0s: -5.577685316351455\n",
      "FDE @1.0s: 0.014470668260911932\n",
      "ADE @1.0s: 0.021401672730783382\n",
      "----------------------------------------------\n",
      "-----------------PH: 4 -------------------\n",
      "FDE Mean @2.0s: 0.21879313416975887\n",
      "ADE Mean @2.0s: 0.10080166010252017\n",
      "KDE Mean @2.0s: -3.9582677566570568\n",
      "FDE @2.0s: 0.1656927524369561\n",
      "ADE @2.0s: 0.07265244240382243\n",
      "----------------------------------------------\n",
      "-----------------PH: 6 -------------------\n",
      "FDE Mean @3.0s: 0.48124106327369537\n",
      "ADE Mean @3.0s: 0.20455084715465008\n",
      "KDE Mean @3.0s: -2.768212012793919\n",
      "FDE @3.0s: 0.36991744855974507\n",
      "ADE @3.0s: 0.1538591151610063\n",
      "----------------------------------------------\n",
      "-----------------PH: 8 -------------------\n",
      "FDE Mean @4.0s: 0.7897925016736143\n",
      "ADE Mean @4.0s: 0.3309282373807616\n",
      "KDE Mean @4.0s: -1.891451489507079\n",
      "FDE @4.0s: 0.61780508431085\n",
      "ADE @4.0s: 0.2535511093237994\n",
      "----------------------------------------------\n",
      "\n",
      "FDE Results for: vel_ee_ped\n",
      "-----------------PH: 2 -------------------\n",
      "FDE Mean @1.0s: 0.05470159146400349\n",
      "ADE Mean @1.0s: 0.04723856023122099\n",
      "KDE Mean @1.0s: -2.693286369409014\n",
      "FDE @1.0s: 0.03272132837594798\n",
      "ADE @1.0s: 0.03440844320849249\n",
      "----------------------------------------------\n",
      "-----------------PH: 4 -------------------\n",
      "FDE Mean @2.0s: 0.235549582909888\n",
      "ADE Mean @2.0s: 0.11606559815399368\n",
      "KDE Mean @2.0s: -2.4601640447400186\n",
      "FDE @2.0s: 0.17398568920641183\n",
      "ADE @2.0s: 0.08409326559182477\n",
      "----------------------------------------------\n",
      "-----------------PH: 6 -------------------\n",
      "FDE Mean @3.0s: 0.4833427705400407\n",
      "ADE Mean @3.0s: 0.21676831990727596\n",
      "KDE Mean @3.0s: -1.7550238928047612\n",
      "FDE @3.0s: 0.3705610422470493\n",
      "ADE @3.0s: 0.16234687699669642\n",
      "----------------------------------------------\n",
      "-----------------PH: 8 -------------------\n",
      "FDE Mean @4.0s: 0.7761647665317681\n",
      "ADE Mean @4.0s: 0.3376368652760976\n",
      "KDE Mean @4.0s: -1.0900967343150951\n",
      "FDE @4.0s: 0.6033992852865975\n",
      "ADE @4.0s: 0.25754615271005243\n",
      "----------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model in ['int_ee_me_ped', 'vel_ee_ped']:\n",
    "    print(f\"FDE Results for: {model}\")\n",
    "    for ph in [2, 4, 6, 8]:\n",
    "        print(f\"-----------------PH: {ph} -------------------\")\n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}*_{ph}_fde_full.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"FDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'fde'].mean()}\")\n",
    "        del perf_df    \n",
    "              \n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}*_{ph}_ade_full.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"ADE Mean @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'ade'].mean()}\")\n",
    "        del perf_df\n",
    "              \n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}*_{ph}_kde_full.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"KDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'kde'].mean()}\")\n",
    "        del perf_df  \n",
    "\n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}*_{ph}_fde_most_likely_z.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"FDE @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'fde'].mean()}\")      \n",
    "        del perf_df\n",
    "              \n",
    "        perf_df = pd.DataFrame()\n",
    "        for f in glob.glob(f\"results/{model}*_{ph}_ade_most_likely_z.csv\"):\n",
    "            dataset_df = pd.read_csv(f)\n",
    "            dataset_df['model'] = model\n",
    "            perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
    "        del perf_df['Unnamed: 0']\n",
    "        print(f\"ADE @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'ade'].mean()}\")      \n",
    "        del perf_df\n",
    "        print(\"----------------------------------------------\")\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:trajectron] *",
   "language": "python",
   "name": "conda-env-trajectron-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}