From 2171dd459a9e8490fd1b871d9aca10bce5494f7a Mon Sep 17 00:00:00 2001 From: Ruben van de Ven Date: Fri, 20 Oct 2023 13:27:04 +0200 Subject: [PATCH] switch to not predict video, but training data --- trap/prediction_server.py | 129 ++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/trap/prediction_server.py b/trap/prediction_server.py index d2b178f..9217c6c 100644 --- a/trap/prediction_server.py +++ b/trap/prediction_server.py @@ -214,13 +214,13 @@ class PredictionServer: prev_run_time = 0 while self.is_running.is_set(): timestep += 1 - this_run_time = time.time() - logger.debug(f'test {prev_run_time - this_run_time}') - time.sleep(max(0, prev_run_time - this_run_time + .5)) - prev_run_time = time.time() - # for timestep in range(init_timestep + 1, eval_scene.timesteps): + + # this_run_time = time.time() + # logger.debug(f'test {prev_run_time - this_run_time}') + # time.sleep(max(0, prev_run_time - this_run_time + .5)) + # prev_run_time = time.time() + - # input_dict = eval_scene.get_clipped_input_dict(timestep, hyperparams['state']) # TODO: see process_data.py on how to create a node, the provide nodes + incoming data columns # data_columns = pd.MultiIndex.from_product([['position', 'velocity', 'acceleration'], ['x', 'y']]) # x = node_values[:, 0] @@ -239,62 +239,66 @@ class PredictionServer: # node_data = pd.DataFrame(data_dict, columns=data_columns) # node = Node(node_type=env.NodeType.PEDESTRIAN, node_id=node_id, data=node_data) - - data = self.trajectory_socket.recv() - frame: Frame = pickle.loads(data) - trajectory_data = frame.trajectories # TODO: properly refractor - # trajectory_data = json.loads(data) - logger.debug(f"Receive {trajectory_data}") + if self.config.predict_training_data: + input_dict = eval_scene.get_clipped_input_dict(timestep, hyperparams['state']) + else: + data = self.trajectory_socket.recv() + frame: Frame = pickle.loads(data) + trajectory_data = frame.trajectories # TODO: properly refractor + # trajectory_data = json.loads(data) + logger.debug(f"Receive {trajectory_data}") - # class FakeNode: - # def __init__(self, node_type: NodeType): - # self.type = node_type + # class FakeNode: + # def __init__(self, node_type: NodeType): + # self.type = node_type - input_dict = {} - for identifier, trajectory in trajectory_data.items(): - # if len(trajectory['history']) < 7: - # # TODO: these trajectories should still be in the output, but without predictions - # continue + input_dict = {} + for identifier, trajectory in trajectory_data.items(): + # if len(trajectory['history']) < 7: + # # TODO: these trajectories should still be in the output, but without predictions + # continue - # TODO: modify this into a mapping function between JS data an the expected Node format - # node = FakeNode(online_env.NodeType.PEDESTRIAN) - history = [[h['x'], h['y']] for h in trajectory['history']] - history = np.array(history) - x = history[:, 0] - y = history[:, 1] - # TODO: calculate dt based on input - vx = derivative_of(x, 0.2) #eval_scene.dt - vy = derivative_of(y, 0.2) - ax = derivative_of(vx, 0.2) - ay = derivative_of(vy, 0.2) + # TODO: modify this into a mapping function between JS data an the expected Node format + # node = FakeNode(online_env.NodeType.PEDESTRIAN) + history = [[h['x'], h['y']] for h in trajectory['history']] + history = np.array(history) + x = history[:, 0] + y = history[:, 1] + # TODO: calculate dt based on input + vx = derivative_of(x, 0.2) #eval_scene.dt + vy = derivative_of(y, 0.2) + ax = derivative_of(vx, 0.2) + ay = derivative_of(vy, 0.2) - data_dict = {('position', 'x'): x[:], - ('position', 'y'): y[:], - ('velocity', 'x'): vx[:], - ('velocity', 'y'): vy[:], - ('acceleration', 'x'): ax[:], - ('acceleration', 'y'): ay[:]} - data_columns = pd.MultiIndex.from_product([['position', 'velocity', 'acceleration'], ['x', 'y']]) + data_dict = {('position', 'x'): x[:], + ('position', 'y'): y[:], + ('velocity', 'x'): vx[:], + ('velocity', 'y'): vy[:], + ('acceleration', 'x'): ax[:], + ('acceleration', 'y'): ay[:]} + data_columns = pd.MultiIndex.from_product([['position', 'velocity', 'acceleration'], ['x', 'y']]) - node_data = pd.DataFrame(data_dict, columns=data_columns) - node = Node( - node_type=online_env.NodeType.PEDESTRIAN, - node_id=identifier, - data=node_data, - first_timestep=timestep - ) + node_data = pd.DataFrame(data_dict, columns=data_columns) + node = Node( + node_type=online_env.NodeType.PEDESTRIAN, + node_id=identifier, + data=node_data, + first_timestep=timestep + ) - input_dict[node] = np.array([x[-1],y[-1],vx[-1],vy[-1],ax[-1],ay[-1]]) + input_dict[node] = np.array([x[-1],y[-1],vx[-1],vy[-1],ax[-1],ay[-1]]) - # print(input_dict) + # print(input_dict) - if not len(input_dict): - # skip if our input is empty - # TODO: we want to send out empty result... + if not len(input_dict): + # skip if our input is empty + # TODO: we want to send out empty result... + # And want to update the network - data = json.dumps({}) - self.prediction_socket.send_string(data) - continue + data = json.dumps({}) + self.prediction_socket.send_string(data) + + continue maps = None if hyperparams['use_map_encoding']: @@ -311,12 +315,14 @@ class PredictionServer: # robot_present_and_future += adjustment start = time.time() - dists, preds = trajectron.incremental_forward(input_dict, - maps, - prediction_horizon=20, # TODO: make variable - num_samples=2, # TODO: make variable - robot_present_and_future=robot_present_and_future, - full_dist=True) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') # prevent deluge of UserWarning from torch's rrn.py + dists, preds = trajectron.incremental_forward(input_dict, + maps, + prediction_horizon=25, # TODO: make variable + num_samples=20, # TODO: make variable + robot_present_and_future=robot_present_and_future, + full_dist=True) end = time.time() logger.debug("took %.2f s (= %.2f Hz) w/ %d nodes and %d edges" % (end - start, 1. / (end - start), len(trajectron.nodes), @@ -365,7 +371,10 @@ class PredictionServer: } data = json.dumps(response) - logger.info(f"Total frame delay = {time.time()-frame.time}s ({len(trajectron.nodes)} nodes & {trajectron.scene_graph.get_num_edges()} edges. Trajectron: {end - start}s)") + if self.config.predict_training_data: + logger.info(f"Frame prediction: {len(trajectron.nodes)} nodes & {trajectron.scene_graph.get_num_edges()} edges. Trajectron: {end - start}s") + else: + logger.info(f"Total frame delay = {time.time()-frame.time}s ({len(trajectron.nodes)} nodes & {trajectron.scene_graph.get_num_edges()} edges. Trajectron: {end - start}s)") self.prediction_socket.send_string(data) logger.info('Stopping')