Trajectron-plus-plus/experiments/nuScenes/NuScenes Quantitative.ipynb
2020-04-05 21:43:49 -04:00

19 KiB

In [1]:
import pandas as pd
import numpy as np
import glob
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker

Vehicles

In [28]:
for model in ['int_ee', 'int_ee_me', 'vel_ee', 'robot']:
    print(f"FDE Results for: {model}")
    for ph in [2, 4, 6, 8]:
        print(f"-----------------PH: {ph} -------------------")
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}_{ph}_fde_full.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        
        print(f"FDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}")
        del perf_df      
        
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}_{ph}_rv_full.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"RB Viols @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].sum() / (len(perf_df['value'][perf_df['type'] == 'full'].index)*2000)}")
        del perf_df

        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}*_{ph}_fde_most_likely_z.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"FDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'ml'].mean()}")      
        del perf_df
              
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}*_{ph}_kde_full.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"KDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}")      
        print("----------------------------------------------")
        del perf_df
    print("")
FDE Results for: int_ee
-----------------PH: 2 -------------------
FDE Mean @1.0s: 0.15211091398370916
RB Viols @1.0s: 0.0024947383125349227
FDE @1.0s: 0.06731242196606527
KDE @1.0s: -4.282745726122304
----------------------------------------------
-----------------PH: 4 -------------------
FDE Mean @2.0s: 0.6665500898672814
RB Viols @2.0s: 0.006711305643509033
FDE @2.0s: 0.4448624482250238
KDE @2.0s: -2.817182897766484
----------------------------------------------
-----------------PH: 6 -------------------
FDE Mean @3.0s: 1.6002716865301074
RB Viols @3.0s: 0.03183521139877072
FDE @3.0s: 1.1269273173570473
KDE @3.0s: -1.672070802164807
----------------------------------------------
-----------------PH: 8 -------------------
FDE Mean @4.0s: 2.9889876939803504
RB Viols @4.0s: 0.08143642515676414
FDE @4.0s: 2.1726876209006143
KDE @4.0s: -0.7623941477599536
----------------------------------------------

FDE Results for: int_ee_me
-----------------PH: 2 -------------------
FDE Mean @1.0s: 0.17146557916606217
RB Viols @1.0s: 0.002893866020984665
FDE @1.0s: 0.06825468723974978
KDE @1.0s: -4.174627329473856
----------------------------------------------
-----------------PH: 4 -------------------
FDE Mean @2.0s: 0.6874934647937203
RB Viols @2.0s: 0.006347814614763767
FDE @2.0s: 0.4473287549407142
KDE @2.0s: -2.7424043655184898
----------------------------------------------
-----------------PH: 6 -------------------
FDE Mean @3.0s: 1.6150508554078604
RB Viols @3.0s: 0.027944558266592166
FDE @3.0s: 1.1370181075818808
KDE @3.0s: -1.616241617749356
----------------------------------------------
-----------------PH: 8 -------------------
FDE Mean @4.0s: 2.9834139311814645
RB Viols @4.0s: 0.07611557086980816
FDE @4.0s: 2.2067347028461923
KDE @4.0s: -0.7050671606779637
----------------------------------------------

FDE Results for: vel_ee_me
-----------------PH: 2 -------------------
FDE Mean @1.0s: 0.21398885662219846
RB Viols @1.0s: 0.0024283075681380767
FDE @1.0s: 0.1792272294232774
KDE @1.0s: 0.8111385940397233
----------------------------------------------
-----------------PH: 4 -------------------
FDE Mean @2.0s: 0.715463329547642
RB Viols @2.0s: 0.006407897187558204
FDE @2.0s: 0.5706283482566946
KDE @2.0s: 0.051893685490453464
----------------------------------------------
-----------------PH: 6 -------------------
FDE Mean @3.0s: 1.5440473025828012
RB Viols @3.0s: 0.02805111131806047
FDE @3.0s: 1.2515989489585615
KDE @3.0s: 0.371638561867866
----------------------------------------------
-----------------PH: 8 -------------------
FDE Mean @4.0s: 2.714255228812044
RB Viols @4.0s: 0.06920216365555348
FDE @4.0s: 2.2400267464847876
KDE @4.0s: 0.8726346089263975
----------------------------------------------

FDE Results for: robot
-----------------PH: 2 -------------------
FDE Mean @1.0s: 0.1295215269389519
RB Viols @1.0s: 0.0026757717999638924
FDE @1.0s: 0.07820393052295552
KDE @1.0s: -3.906838146881899
----------------------------------------------
-----------------PH: 4 -------------------
FDE Mean @2.0s: 0.45962341869964574
RB Viols @2.0s: 0.0053363964614551365
FDE @2.0s: 0.3403511030418785
KDE @2.0s: -2.7593676749477294
----------------------------------------------
-----------------PH: 6 -------------------
FDE Mean @3.0s: 1.02267032097404
RB Viols @3.0s: 0.016484509839321176
FDE @3.0s: 0.805915047871091
KDE @3.0s: -1.7502450775203158
----------------------------------------------
-----------------PH: 8 -------------------
FDE Mean @4.0s: 1.8380306576706953
RB Viols @4.0s: 0.042144791478606246
FDE @4.0s: 1.4979755853506684
KDE @4.0s: -0.9291549495198915
----------------------------------------------

In [38]:
for model in ['int_ee_me_no_ego', 'robot']:
    print(f"FDE Results for: {model}")
    for ph in [2, 4, 6, 8]:
        print(f"-----------------PH: {ph} -------------------")
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}_{ph}_fde_full.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        
        print(f"FDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}")
        del perf_df      
        
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}_{ph}_rv_full.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"RB Viols @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].sum() / (len(perf_df['value'][perf_df['type'] == 'full'].index)*2000)}")
        del perf_df

        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}*_{ph}_fde_most_likely_z.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"FDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'ml'].mean()}")      
        del perf_df
              
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}*_{ph}_kde_full.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"KDE @{ph*0.5}s: {perf_df['value'][perf_df['type'] == 'full'].mean()}")      
        print("----------------------------------------------")
        del perf_df
    print("")
FDE Results for: int_ee_me_no_ego
-----------------PH: 2 -------------------
FDE Mean @1.0s: 0.16815279412540554
RB Viols @1.0s: 0.002895014589929844
FDE @1.0s: 0.06937045846177256
KDE @1.0s: -4.262019931215572
----------------------------------------------
-----------------PH: 4 -------------------
FDE Mean @2.0s: 0.6655379721067188
RB Viols @2.0s: 0.006153364996585336
FDE @2.0s: 0.4359008486971371
KDE @2.0s: -2.856656149202157
----------------------------------------------
-----------------PH: 6 -------------------
FDE Mean @3.0s: 1.546091556287448
RB Viols @3.0s: 0.027780530204259017
FDE @3.0s: 1.0896218245514429
KDE @3.0s: -1.7563896369106704
----------------------------------------------
-----------------PH: 8 -------------------
FDE Mean @4.0s: 2.8358865412257397
RB Viols @4.0s: 0.07581256596510834
FDE @4.0s: 2.0939721352439022
KDE @4.0s: -0.8690706892091696
----------------------------------------------

FDE Results for: robot
-----------------PH: 2 -------------------
FDE Mean @1.0s: 0.1295215269389519
RB Viols @1.0s: 0.0026757717999638924
FDE @1.0s: 0.07820393052295552
KDE @1.0s: -3.906838146881899
----------------------------------------------
-----------------PH: 4 -------------------
FDE Mean @2.0s: 0.45962341869964574
RB Viols @2.0s: 0.0053363964614551365
FDE @2.0s: 0.3403511030418785
KDE @2.0s: -2.7593676749477294
----------------------------------------------
-----------------PH: 6 -------------------
FDE Mean @3.0s: 1.02267032097404
RB Viols @3.0s: 0.016484509839321176
FDE @3.0s: 0.805915047871091
KDE @3.0s: -1.7502450775203158
----------------------------------------------
-----------------PH: 8 -------------------
FDE Mean @4.0s: 1.8380306576706953
RB Viols @4.0s: 0.042144791478606246
FDE @4.0s: 1.4979755853506684
KDE @4.0s: -0.9291549495198915
----------------------------------------------

Pedestrians

In [40]:
for model in ['int_ee_me_ped', 'vel_ee_ped']:
    print(f"FDE Results for: {model}")
    for ph in [2, 4, 6, 8]:
        print(f"-----------------PH: {ph} -------------------")
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}*_{ph}_fde_full.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"FDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'fde'].mean()}")
        del perf_df    
              
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}*_{ph}_ade_full.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"ADE Mean @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'ade'].mean()}")
        del perf_df
              
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}*_{ph}_kde_full.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"KDE Mean @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'kde'].mean()}")
        del perf_df  

        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}*_{ph}_fde_most_likely_z.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"FDE @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'fde'].mean()}")      
        del perf_df
              
        perf_df = pd.DataFrame()
        for f in glob.glob(f"results/{model}*_{ph}_ade_most_likely_z.csv"):
            dataset_df = pd.read_csv(f)
            dataset_df['model'] = model
            perf_df = perf_df.append(dataset_df, ignore_index=True)
        del perf_df['Unnamed: 0']
        print(f"ADE @{ph*0.5}s: {perf_df['value'][perf_df['metric'] == 'ade'].mean()}")      
        del perf_df
        print("----------------------------------------------")
    print("")
FDE Results for: int_ee_me_ped
-----------------PH: 2 -------------------
FDE Mean @1.0s: 0.03182535279935429
ADE Mean @1.0s: 0.034975306849922005
KDE Mean @1.0s: -5.577685316351455
FDE @1.0s: 0.014470668260911932
ADE @1.0s: 0.021401672730783382
----------------------------------------------
-----------------PH: 4 -------------------
FDE Mean @2.0s: 0.21879313416975887
ADE Mean @2.0s: 0.10080166010252017
KDE Mean @2.0s: -3.9582677566570568
FDE @2.0s: 0.1656927524369561
ADE @2.0s: 0.07265244240382243
----------------------------------------------
-----------------PH: 6 -------------------
FDE Mean @3.0s: 0.48124106327369537
ADE Mean @3.0s: 0.20455084715465008
KDE Mean @3.0s: -2.768212012793919
FDE @3.0s: 0.36991744855974507
ADE @3.0s: 0.1538591151610063
----------------------------------------------
-----------------PH: 8 -------------------
FDE Mean @4.0s: 0.7897925016736143
ADE Mean @4.0s: 0.3309282373807616
KDE Mean @4.0s: -1.891451489507079
FDE @4.0s: 0.61780508431085
ADE @4.0s: 0.2535511093237994
----------------------------------------------

FDE Results for: vel_ee_ped
-----------------PH: 2 -------------------
FDE Mean @1.0s: 0.05470159146400349
ADE Mean @1.0s: 0.04723856023122099
KDE Mean @1.0s: -2.693286369409014
FDE @1.0s: 0.03272132837594798
ADE @1.0s: 0.03440844320849249
----------------------------------------------
-----------------PH: 4 -------------------
FDE Mean @2.0s: 0.235549582909888
ADE Mean @2.0s: 0.11606559815399368
KDE Mean @2.0s: -2.4601640447400186
FDE @2.0s: 0.17398568920641183
ADE @2.0s: 0.08409326559182477
----------------------------------------------
-----------------PH: 6 -------------------
FDE Mean @3.0s: 0.4833427705400407
ADE Mean @3.0s: 0.21676831990727596
KDE Mean @3.0s: -1.7550238928047612
FDE @3.0s: 0.3705610422470493
ADE @3.0s: 0.16234687699669642
----------------------------------------------
-----------------PH: 8 -------------------
FDE Mean @4.0s: 0.7761647665317681
ADE Mean @4.0s: 0.3376368652760976
KDE Mean @4.0s: -1.0900967343150951
FDE @4.0s: 0.6033992852865975
ADE @4.0s: 0.25754615271005243
----------------------------------------------

In [ ]: