From cc5b407b2f420f268942f9929a8781437967c7a1 Mon Sep 17 00:00:00 2001 From: Ana Runova Date: Fri, 3 Mar 2023 14:57:30 -0500 Subject: [PATCH 1/2] fixed the parser for actor data - made it more readable and compatible with panda data frames --- src/parser.py | 34 +++++++++++++++++++--------------- src/utils.py | 11 ++++++++--- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/parser.py b/src/parser.py index fff2043..a3911d4 100644 --- a/src/parser.py +++ b/src/parser.py @@ -109,7 +109,7 @@ def parse_custom_actor( def parse_actor_location_rotation( data: Dict[str, Any], data_line: str, - title: Optional[str] = "", + actors_key: str, t: Optional[int] = 0, ): Id: int = int(data_line[: data_line.find("Location")]) @@ -118,12 +118,14 @@ def parse_actor_location_rotation( open_1: int = close_0 + 2 + len("Rotation") # always " Rotation " loc_data: tuple = eval(data_line[open_0:close_0]) rot_data: tuple = eval(data_line[open_1:]) - if Id not in data[title]: - data[title][Id] = {"Time": [], "Location": [], "Rotation": []} - data[title][Id]["Time"].append(t) - data[title][Id]["Location"].append(loc_data) - data[title][Id]["Rotation"].append(rot_data) - + if len(data[actors_key]["Id"]) <= t: + data[actors_key]["Id"].append([]) + data[actors_key]["Location"].append({}) + data[actors_key]["Rotation"].append({}) + data[actors_key]["Id"][t].append(Id) + data[actors_key]["Location"][t][Id] = loc_data + data[actors_key]["Rotation"][t][Id] = rot_data + def validate(data: Dict[str, Any], L: Optional[int] = None) -> None: # verify the data structure is reasonable @@ -239,6 +241,9 @@ def parse_file( actors_key: str = "Actors" data[actors_key] = {} + data[actors_key]["Id"] = [] + data[actors_key]["Location"] = [] + data[actors_key]["Rotation"] = [] with open(path, "r") as f: start_t: float = time.time() @@ -273,13 +278,12 @@ def parse_file( data_line: str = line.strip(Carla_Actor).strip("\n") if "Location:" not in data_line or "Rotation" not in data_line: continue # don't care about state, light, animation, etc. - if "TimestampCarla" in data: - t = data["TimestampCarla"][_no_title_key][-1] # get carla time - else: - t = 0 - parse_actor_location_rotation(data, data_line, title="Actors", t=t) - if debug: - validate(data) + + t = len(data["TimeElapsed"]) - 1 # get carla time + #print('!', t) + + parse_actor_location_rotation(data, data_line, actors_key, t=t) + # print status if i % 500 == 0: @@ -316,4 +320,4 @@ def cache_data(data: Dict[str, Any], filename: str) -> None: filename = f"{os.path.join(cache_dir, actual_name)}.pkl" with open(filename, "wb") as filehandler: pickle.dump(data, filehandler) - print(f"cached data to {filename}") + print(f"cached data to {filename}") \ No newline at end of file diff --git a/src/utils.py b/src/utils.py index 67d6b9f..b01eaae 100644 --- a/src/utils.py +++ b/src/utils.py @@ -110,7 +110,9 @@ def cleanup_data_line(data_line: str) -> str: def convert_to_list(data: Dict[str, np.ndarray or dict]) -> Dict[str, list or dict]: list_data = {} for k in data.keys(): - if isinstance(data[k], dict): + if isinstance(data[k], list): + list_data[k] = data[k] + elif isinstance(data[k], dict): list_data[k] = convert_to_list(data[k]) else: assert isinstance(data[k], np.ndarray) @@ -252,8 +254,11 @@ def convert_to_df(data: Dict[str, Any]) -> pd.DataFrame: data = convert_to_list(data) data = flatten_dict(data) lens = [len(x) for x in data.values()] - assert min(lens) == max(lens) # all lengths are equal! - # NOTE: pandas can't haneld high dimensional np arrays, so we just use lists + # all lengths are equal! + if not min(lens) == max(lens): + import ipdb; ipdb.set_trace() + + # NOTE: pandas can't handle high dimensional np arrays, so we just use lists df = pd.DataFrame.from_dict(data) print(f"created DReyeVR df in {time.time() - start_t:.3f}s") return df From 32d5ba780d68d1bac0339db6b6c3949aa6704fd7 Mon Sep 17 00:00:00 2001 From: Ana Runova Date: Fri, 3 Mar 2023 15:17:04 -0500 Subject: [PATCH 2/2] added a simple test --- simple_test.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 simple_test.py diff --git a/simple_test.py b/simple_test.py new file mode 100644 index 0000000..1e28a7a --- /dev/null +++ b/simple_test.py @@ -0,0 +1,55 @@ +from typing import Dict, Optional, List +from src.parser import parse_file +from src.utils import get_good_idxs, fill_gaps, smooth_arr, compute_YP, convert_to_df, flatten_dict +import numpy as np +import argparse +from src.visualizer import ( + set_results_dir, +) + + +def main(filename: str, results_dir: str, vlines: Optional[List[float]] = None): + set_results_dir(results_dir) + + """parse the file""" + data: Dict[str, np.ndarray or dict] = parse_file(filename) + print(data.keys()) + # Convert to pandas dataframe + import pandas as pd + actors_data = {} + actors_data["TimeElapsed"] = data["TimeElapsed"] + actors_data["Actors"] = {} + actors_data["Actors"]["Location"] = data["Actors"]["Location"] + + + + data_frame = convert_to_df(data) + actors_df = convert_to_df(actors_data) + + # Display the frame + from IPython.display import display + from tabulate import tabulate + print(data_frame.keys()) + print(tabulate(actors_df, headers = 'keys', tablefmt = 'fancy_grid')) + + + +if __name__ == "__main__": + argparser = argparse.ArgumentParser(description="DReyeVR recording parser") + argparser.add_argument( + "-f", + "--file", + metavar="P", + type=str, + help="path of the (human readable) recording file", + ) + argparser.add_argument( + "-o", + "--out", + default="results", + type=str, + help="path of the results folder", + ) + args = argparser.parse_args() + + main(args.file, args.out) \ No newline at end of file