Skip to content

Commit 13463cf

Browse files
committed
fix: preserve scene identifiers when reading CSV files
Read token and prefix columns as strings to prevent numeric inference from altering scene identifiers.
1 parent fe632c3 commit 13463cf

4 files changed

Lines changed: 7 additions & 7 deletions

File tree

‎projects/AlgEngine/mmdet3d_plugin/datasets/navsim_openscene_synthetic.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ def load_syntheitc_infos(self):
5353
plan_dfs = {}
5454
for folder_name in self.folder_names:
5555
# Load plan_idx file
56-
plan_df = pd.read_csv(self.syn_pdm_path + f"/{folder_name}/plan_idx.csv")
56+
plan_df = pd.read_csv(self.syn_pdm_path + f"/{folder_name}/plan_idx.csv", dtype={'prefix': str})
5757
plan_dfs[folder_name] = plan_df
5858

5959
folder_data_infos = []

‎projects/AlgEngine/scripts/rare_case_sampling_by_pdms.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929

3030
def load_pdm_result(path: str) -> pd.DataFrame:
3131
"""Load a PDM evaluation CSV and return a DataFrame."""
32-
df = pd.read_csv(path)
32+
df = pd.read_csv(path, dtype={'token': str})
3333
required_cols = {"token", "no_at_fault_collisions",
3434
"drivable_area_compliance", "ego_progress"}
3535
missing = required_cols - set(df.columns)

‎projects/SimEngine/scripts/merge_simulation_results.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ def merge_plan_csv():
3131
all_dfs = []
3232
for i in range(8):
3333
split_csv = os.path.join(merged_base, f"split_{i}/plan_traj/plan_idx.csv")
34-
# Read all rows except header and last row (overall_average)
35-
df_split = pd.read_csv(split_csv)
34+
# Preserve scene identifiers that resemble numbers or scientific notation.
35+
df_split = pd.read_csv(split_csv, dtype={'prefix': str})
3636
all_dfs.append(df_split)
3737
merged_df = pd.concat(all_dfs, ignore_index=True)
3838
merged_df.to_csv(merged_csv_path, index=False)
@@ -47,7 +47,7 @@ def merge_plan_csv():
4747
all_dfs = []
4848
for i in range(8):
4949
split_csv = os.path.join(merged_base, f"split_{i}/WE_output/openscene_format/all_scenes_pdm_averages_{REACT_TYPE}.csv")
50-
df_split = pd.read_csv(split_csv, skipfooter=1, engine='python')
50+
df_split = pd.read_csv(split_csv, skipfooter=1, engine='python', dtype={'token': str})
5151
all_dfs.append(df_split)
5252
merged_df = pd.concat(all_dfs, ignore_index=True)
5353

@@ -66,7 +66,7 @@ def merge_plan_csv():
6666
# skip if dense reward not computed
6767
if not os.path.exists(split_csv):
6868
continue
69-
df_split = pd.read_csv(split_csv)
69+
df_split = pd.read_csv(split_csv, dtype={'token': str})
7070
all_dfs.append(df_split)
7171
if len(all_dfs) > 0:
7272
merged_df = pd.concat(all_dfs, ignore_index=True)

‎projects/SimEngine/worldengine/manager/metric_manager.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -399,7 +399,7 @@ def save_pdm_scores(self):
399399
# Check if CSV already exists for resume functionality
400400
new_score_row = self.pdm_scores_list[-1]
401401
if os.path.exists(averages_csv_path):
402-
existing_df = pd.read_csv(averages_csv_path)
402+
existing_df = pd.read_csv(averages_csv_path, dtype={'token': str})
403403
# Remove the last row (overall_average)
404404
if len(existing_df) > 0 and existing_df.iloc[-1]['token'] == 'overall_average':
405405
existing_df = existing_df.iloc[:-1]

0 commit comments

Comments
 (0)