Trajectron-plus-plus/experiments/nuScenes/NuScenes Quantitative.ipynb

457 lines
19 KiB
Text
Raw Normal View History

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import glob\n",
"import matplotlib.pyplot as plt\n",
"import matplotlib.ticker as ticker"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Vehicles"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"FDE Results for: int_ee\n",
"-----------------PH: 2 -------------------\n",
"FDE Mean @1.0s: 0.15211091398370916\n",
"RB Viols @1.0s: 0.0024947383125349227\n",
"FDE @1.0s: 0.06731242196606527\n",
"KDE @1.0s: -4.282745726122304\n",
"----------------------------------------------\n",
"-----------------PH: 4 -------------------\n",
"FDE Mean @2.0s: 0.6665500898672814\n",
"RB Viols @2.0s: 0.006711305643509033\n",
"FDE @2.0s: 0.4448624482250238\n",
"KDE @2.0s: -2.817182897766484\n",
"----------------------------------------------\n",
"-----------------PH: 6 -------------------\n",
"FDE Mean @3.0s: 1.6002716865301074\n",
"RB Viols @3.0s: 0.03183521139877072\n",
"FDE @3.0s: 1.1269273173570473\n",
"KDE @3.0s: -1.672070802164807\n",
"----------------------------------------------\n",
"-----------------PH: 8 -------------------\n",
"FDE Mean @4.0s: 2.9889876939803504\n",
"RB Viols @4.0s: 0.08143642515676414\n",
"FDE @4.0s: 2.1726876209006143\n",
"KDE @4.0s: -0.7623941477599536\n",
"----------------------------------------------\n",
"\n",
"FDE Results for: int_ee_me\n",
"-----------------PH: 2 -------------------\n",
"FDE Mean @1.0s: 0.17146557916606217\n",
"RB Viols @1.0s: 0.002893866020984665\n",
"FDE @1.0s: 0.06825468723974978\n",
"KDE @1.0s: -4.174627329473856\n",
"----------------------------------------------\n",
"-----------------PH: 4 -------------------\n",
"FDE Mean @2.0s: 0.6874934647937203\n",
"RB Viols @2.0s: 0.006347814614763767\n",
"FDE @2.0s: 0.4473287549407142\n",
"KDE @2.0s: -2.7424043655184898\n",
"----------------------------------------------\n",
"-----------------PH: 6 -------------------\n",
"FDE Mean @3.0s: 1.6150508554078604\n",
"RB Viols @3.0s: 0.027944558266592166\n",
"FDE @3.0s: 1.1370181075818808\n",
"KDE @3.0s: -1.616241617749356\n",
"----------------------------------------------\n",
"-----------------PH: 8 -------------------\n",
"FDE Mean @4.0s: 2.9834139311814645\n",
"RB Viols @4.0s: 0.07611557086980816\n",
"FDE @4.0s: 2.2067347028461923\n",
"KDE @4.0s: -0.7050671606779637\n",
"----------------------------------------------\n",
"\n",
"FDE Results for: vel_ee_me\n",
"-----------------PH: 2 -------------------\n",
"FDE Mean @1.0s: 0.21398885662219846\n",
"RB Viols @1.0s: 0.0024283075681380767\n",
"FDE @1.0s: 0.1792272294232774\n",
"KDE @1.0s: 0.8111385940397233\n",
"----------------------------------------------\n",
"-----------------PH: 4 -------------------\n",
"FDE Mean @2.0s: 0.715463329547642\n",
"RB Viols @2.0s: 0.006407897187558204\n",
"FDE @2.0s: 0.5706283482566946\n",
"KDE @2.0s: 0.051893685490453464\n",
"----------------------------------------------\n",
"-----------------PH: 6 -------------------\n",
"FDE Mean @3.0s: 1.5440473025828012\n",
"RB Viols @3.0s: 0.02805111131806047\n",
"FDE @3.0s: 1.2515989489585615\n",
"KDE @3.0s: 0.371638561867866\n",
"----------------------------------------------\n",
"-----------------PH: 8 -------------------\n",
"FDE Mean @4.0s: 2.714255228812044\n",
"RB Viols @4.0s: 0.06920216365555348\n",
"FDE @4.0s: 2.2400267464847876\n",
"KDE @4.0s: 0.8726346089263975\n",
"----------------------------------------------\n",
"\n",
"FDE Results for: robot\n",
"-----------------PH: 2 -------------------\n",
"FDE Mean @1.0s: 0.1295215269389519\n",
"RB Viols @1.0s: 0.0026757717999638924\n",
"FDE @1.0s: 0.07820393052295552\n",
"KDE @1.0s: -3.906838146881899\n",
"----------------------------------------------\n",
"-----------------PH: 4 -------------------\n",
"FDE Mean @2.0s: 0.45962341869964574\n",
"RB Viols @2.0s: 0.0053363964614551365\n",
"FDE @2.0s: 0.3403511030418785\n",
"KDE @2.0s: -2.7593676749477294\n",
"----------------------------------------------\n",
"-----------------PH: 6 -------------------\n",
"FDE Mean @3.0s: 1.02267032097404\n",
"RB Viols @3.0s: 0.016484509839321176\n",
"FDE @3.0s: 0.805915047871091\n",
"KDE @3.0s: -1.7502450775203158\n",
"----------------------------------------------\n",
"-----------------PH: 8 -------------------\n",
"FDE Mean @4.0s: 1.8380306576706953\n",
"RB Viols @4.0s: 0.042144791478606246\n",
"FDE @4.0s: 1.4979755853506684\n",
"KDE @4.0s: -0.9291549495198915\n",
"----------------------------------------------\n",
"\n"
]
}
],
"source": [
"for model in ['int_ee', 'int_ee_me', 'vel_ee', 'robot']:\n",
" print(f\"FDE Results for: {model}\")\n",
" for ph in [2, 4, 6, 8]:\n",
" print(f\"-----------------PH: {ph} -------------------\")\n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}_{ph}_fde_full.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" \n",
" print(f\"FDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}\")\n",
" del perf_df \n",
" \n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}_{ph}_rv_full.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"RB Viols @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].sum() / (len(perf_df['value'][perf_df['type'] == 'full'].index)*2000)}\")\n",
" del perf_df\n",
"\n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}*_{ph}_fde_most_likely_z.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"FDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'ml'].mean()}\") \n",
" del perf_df\n",
" \n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}*_{ph}_kde_full.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"KDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}\") \n",
" print(\"----------------------------------------------\")\n",
" del perf_df\n",
" print(\"\")"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"FDE Results for: int_ee_me_no_ego\n",
"-----------------PH: 2 -------------------\n",
"FDE Mean @1.0s: 0.16815279412540554\n",
"RB Viols @1.0s: 0.002895014589929844\n",
"FDE @1.0s: 0.06937045846177256\n",
"KDE @1.0s: -4.262019931215572\n",
"----------------------------------------------\n",
"-----------------PH: 4 -------------------\n",
"FDE Mean @2.0s: 0.6655379721067188\n",
"RB Viols @2.0s: 0.006153364996585336\n",
"FDE @2.0s: 0.4359008486971371\n",
"KDE @2.0s: -2.856656149202157\n",
"----------------------------------------------\n",
"-----------------PH: 6 -------------------\n",
"FDE Mean @3.0s: 1.546091556287448\n",
"RB Viols @3.0s: 0.027780530204259017\n",
"FDE @3.0s: 1.0896218245514429\n",
"KDE @3.0s: -1.7563896369106704\n",
"----------------------------------------------\n",
"-----------------PH: 8 -------------------\n",
"FDE Mean @4.0s: 2.8358865412257397\n",
"RB Viols @4.0s: 0.07581256596510834\n",
"FDE @4.0s: 2.0939721352439022\n",
"KDE @4.0s: -0.8690706892091696\n",
"----------------------------------------------\n",
"\n",
"FDE Results for: robot\n",
"-----------------PH: 2 -------------------\n",
"FDE Mean @1.0s: 0.1295215269389519\n",
"RB Viols @1.0s: 0.0026757717999638924\n",
"FDE @1.0s: 0.07820393052295552\n",
"KDE @1.0s: -3.906838146881899\n",
"----------------------------------------------\n",
"-----------------PH: 4 -------------------\n",
"FDE Mean @2.0s: 0.45962341869964574\n",
"RB Viols @2.0s: 0.0053363964614551365\n",
"FDE @2.0s: 0.3403511030418785\n",
"KDE @2.0s: -2.7593676749477294\n",
"----------------------------------------------\n",
"-----------------PH: 6 -------------------\n",
"FDE Mean @3.0s: 1.02267032097404\n",
"RB Viols @3.0s: 0.016484509839321176\n",
"FDE @3.0s: 0.805915047871091\n",
"KDE @3.0s: -1.7502450775203158\n",
"----------------------------------------------\n",
"-----------------PH: 8 -------------------\n",
"FDE Mean @4.0s: 1.8380306576706953\n",
"RB Viols @4.0s: 0.042144791478606246\n",
"FDE @4.0s: 1.4979755853506684\n",
"KDE @4.0s: -0.9291549495198915\n",
"----------------------------------------------\n",
"\n"
]
}
],
"source": [
"for model in ['int_ee_me_no_ego', 'robot']:\n",
" print(f\"FDE Results for: {model}\")\n",
" for ph in [2, 4, 6, 8]:\n",
" print(f\"-----------------PH: {ph} -------------------\")\n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}_{ph}_fde_full.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" \n",
" print(f\"FDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}\")\n",
" del perf_df \n",
" \n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}_{ph}_rv_full.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"RB Viols @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].sum() / (len(perf_df['value'][perf_df['type'] == 'full'].index)*2000)}\")\n",
" del perf_df\n",
"\n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}*_{ph}_fde_most_likely_z.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"FDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'ml'].mean()}\") \n",
" del perf_df\n",
" \n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}*_{ph}_kde_full.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"KDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}\") \n",
" print(\"----------------------------------------------\")\n",
" del perf_df\n",
" print(\"\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Pedestrians"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"FDE Results for: int_ee_me_ped\n",
"-----------------PH: 2 -------------------\n",
"FDE Mean @1.0s: 0.03182535279935429\n",
"ADE Mean @1.0s: 0.034975306849922005\n",
"KDE Mean @1.0s: -5.577685316351455\n",
"FDE @1.0s: 0.014470668260911932\n",
"ADE @1.0s: 0.021401672730783382\n",
"----------------------------------------------\n",
"-----------------PH: 4 -------------------\n",
"FDE Mean @2.0s: 0.21879313416975887\n",
"ADE Mean @2.0s: 0.10080166010252017\n",
"KDE Mean @2.0s: -3.9582677566570568\n",
"FDE @2.0s: 0.1656927524369561\n",
"ADE @2.0s: 0.07265244240382243\n",
"----------------------------------------------\n",
"-----------------PH: 6 -------------------\n",
"FDE Mean @3.0s: 0.48124106327369537\n",
"ADE Mean @3.0s: 0.20455084715465008\n",
"KDE Mean @3.0s: -2.768212012793919\n",
"FDE @3.0s: 0.36991744855974507\n",
"ADE @3.0s: 0.1538591151610063\n",
"----------------------------------------------\n",
"-----------------PH: 8 -------------------\n",
"FDE Mean @4.0s: 0.7897925016736143\n",
"ADE Mean @4.0s: 0.3309282373807616\n",
"KDE Mean @4.0s: -1.891451489507079\n",
"FDE @4.0s: 0.61780508431085\n",
"ADE @4.0s: 0.2535511093237994\n",
"----------------------------------------------\n",
"\n",
"FDE Results for: vel_ee_ped\n",
"-----------------PH: 2 -------------------\n",
"FDE Mean @1.0s: 0.05470159146400349\n",
"ADE Mean @1.0s: 0.04723856023122099\n",
"KDE Mean @1.0s: -2.693286369409014\n",
"FDE @1.0s: 0.03272132837594798\n",
"ADE @1.0s: 0.03440844320849249\n",
"----------------------------------------------\n",
"-----------------PH: 4 -------------------\n",
"FDE Mean @2.0s: 0.235549582909888\n",
"ADE Mean @2.0s: 0.11606559815399368\n",
"KDE Mean @2.0s: -2.4601640447400186\n",
"FDE @2.0s: 0.17398568920641183\n",
"ADE @2.0s: 0.08409326559182477\n",
"----------------------------------------------\n",
"-----------------PH: 6 -------------------\n",
"FDE Mean @3.0s: 0.4833427705400407\n",
"ADE Mean @3.0s: 0.21676831990727596\n",
"KDE Mean @3.0s: -1.7550238928047612\n",
"FDE @3.0s: 0.3705610422470493\n",
"ADE @3.0s: 0.16234687699669642\n",
"----------------------------------------------\n",
"-----------------PH: 8 -------------------\n",
"FDE Mean @4.0s: 0.7761647665317681\n",
"ADE Mean @4.0s: 0.3376368652760976\n",
"KDE Mean @4.0s: -1.0900967343150951\n",
"FDE @4.0s: 0.6033992852865975\n",
"ADE @4.0s: 0.25754615271005243\n",
"----------------------------------------------\n",
"\n"
]
}
],
"source": [
"for model in ['int_ee_me_ped', 'vel_ee_ped']:\n",
" print(f\"FDE Results for: {model}\")\n",
" for ph in [2, 4, 6, 8]:\n",
" print(f\"-----------------PH: {ph} -------------------\")\n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}*_{ph}_fde_full.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"FDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'fde'].mean()}\")\n",
" del perf_df \n",
" \n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}*_{ph}_ade_full.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"ADE Mean @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'ade'].mean()}\")\n",
" del perf_df\n",
" \n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}*_{ph}_kde_full.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"KDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'kde'].mean()}\")\n",
" del perf_df \n",
"\n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}*_{ph}_fde_most_likely_z.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"FDE @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'fde'].mean()}\") \n",
" del perf_df\n",
" \n",
" perf_df = pd.DataFrame()\n",
" for f in glob.glob(f\"results/{model}*_{ph}_ade_most_likely_z.csv\"):\n",
" dataset_df = pd.read_csv(f)\n",
" dataset_df['model'] = model\n",
" perf_df = perf_df.append(dataset_df, ignore_index=True)\n",
" del perf_df['Unnamed: 0']\n",
" print(f\"ADE @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'ade'].mean()}\") \n",
" del perf_df\n",
" print(\"----------------------------------------------\")\n",
" print(\"\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:trajectron] *",
"language": "python",
"name": "conda-env-trajectron-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 2
}