From e2c7038cffcadb822c0ee0936a3d0a48ae32e7b5 Mon Sep 17 00:00:00 2001 From: Bartosz Grabowski <58475557+bartosz-grabowski@users.noreply.github.com> Date: Tue, 18 Feb 2025 23:49:32 +0100 Subject: [PATCH 1/4] Fix pd.DataFrame indexing in convert_tables_to_dicts() Resolves: #8201 Signed-off-by: Bartosz Grabowski <58475557+bartosz-grabowski@users.noreply.github.com> --- monai/data/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/data/utils.py b/monai/data/utils.py index d03dbd3234..b92bd57f85 100644 --- a/monai/data/utils.py +++ b/monai/data/utils.py @@ -1484,7 +1484,7 @@ def convert_tables_to_dicts( rows.append(i) # convert to a list of dictionaries corresponding to every row - data_ = df.loc[rows] if col_names is None else df.loc[rows, col_names] + data_ = df.iloc[rows] if col_names is None else df.iloc[rows][col_names] if isinstance(col_types, dict): # fill default values for NaN defaults = {k: v["default"] for k, v in col_types.items() if v is not None and v.get("default") is not None} From 4cfd327ecffbf26d4a8e3c4b0de9f07db38f468b Mon Sep 17 00:00:00 2001 From: Bartosz Grabowski <58475557+bartosz-grabowski@users.noreply.github.com> Date: Tue, 18 Feb 2025 23:53:07 +0100 Subject: [PATCH 2/4] Add unittest with DataFrame subset for CSVDataset() Signed-off-by: Bartosz Grabowski <58475557+bartosz-grabowski@users.noreply.github.com> --- tests/data/test_csv_dataset.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/data/test_csv_dataset.py b/tests/data/test_csv_dataset.py index 71be4fdd22..276a0dae94 100644 --- a/tests/data/test_csv_dataset.py +++ b/tests/data/test_csv_dataset.py @@ -179,6 +179,23 @@ def prepare_csv_file(data, filepath): }, ) + # test pre-loaded DataFrame subset + df = pd.read_csv(filepath1) + df_subset = df.iloc[[1, 3, 4]] + dataset = CSVDataset(src=df_subset) + self.assertDictEqual( + {k: round(v, 4) if not isinstance(v, str) else v for k, v in dataset[2].items()}, + { + "subject_id": "s000004", + "label": 9, + "image": "./imgs/s000004.png", + "ehr_0": 6.4275, + "ehr_1": 6.2549, + "ehr_2": 5.9765, + }, + ) + self.assertEqual(len(dataset), 3) + # test pre-loaded multiple DataFrames, join tables with kwargs dfs = [pd.read_csv(i) for i in filepaths] dataset = CSVDataset(src=dfs, on="subject_id") From e46423932b78a8991a5889d7f5953b832fb3bcf6 Mon Sep 17 00:00:00 2001 From: Bartosz Grabowski <58475557+bartosz-grabowski@users.noreply.github.com> Date: Wed, 19 Feb 2025 01:31:17 +0100 Subject: [PATCH 3/4] Fix indexing in convert_tables_to_dicts() for col_groups != None Also updated the unittest to pass after the fix and not before. Signed-off-by: Bartosz Grabowski <58475557+bartosz-grabowski@users.noreply.github.com> --- monai/data/utils.py | 2 +- tests/data/test_csv_dataset.py | 17 +++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/monai/data/utils.py b/monai/data/utils.py index b92bd57f85..9915e0f363 100644 --- a/monai/data/utils.py +++ b/monai/data/utils.py @@ -1500,7 +1500,7 @@ def convert_tables_to_dicts( if col_groups is not None: groups: dict[str, list] = {} for name, cols in col_groups.items(): - groups[name] = df.loc[rows, cols].values + groups[name] = df.iloc[rows][cols].values # invert items of groups to every row of data data = [dict(d, **{k: v[i] for k, v in groups.items()}) for i, d in enumerate(data)] diff --git a/tests/data/test_csv_dataset.py b/tests/data/test_csv_dataset.py index 276a0dae94..8f0d333408 100644 --- a/tests/data/test_csv_dataset.py +++ b/tests/data/test_csv_dataset.py @@ -182,19 +182,12 @@ def prepare_csv_file(data, filepath): # test pre-loaded DataFrame subset df = pd.read_csv(filepath1) df_subset = df.iloc[[1, 3, 4]] - dataset = CSVDataset(src=df_subset) - self.assertDictEqual( - {k: round(v, 4) if not isinstance(v, str) else v for k, v in dataset[2].items()}, - { - "subject_id": "s000004", - "label": 9, - "image": "./imgs/s000004.png", - "ehr_0": 6.4275, - "ehr_1": 6.2549, - "ehr_2": 5.9765, - }, - ) + dataset = CSVDataset(src=df_subset, col_groups={"ehr": [f"ehr_{i}" for i in range(3)]}) self.assertEqual(len(dataset), 3) + np.testing.assert_allclose( + [round(i, 4) for i in dataset[1]["ehr"]], + [3.3333, 3.2353, 3.4000], + ) # test pre-loaded multiple DataFrames, join tables with kwargs dfs = [pd.read_csv(i) for i in filepaths] From 7c6d06e41afe34d356a5271bb554edbee62979b8 Mon Sep 17 00:00:00 2001 From: Bartosz Grabowski <58475557+bartosz-grabowski@users.noreply.github.com> Date: Thu, 20 Feb 2025 17:05:25 +0100 Subject: [PATCH 4/4] Black formatting Signed-off-by: Bartosz Grabowski <58475557+bartosz-grabowski@users.noreply.github.com> --- tests/data/test_csv_dataset.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/data/test_csv_dataset.py b/tests/data/test_csv_dataset.py index 8f0d333408..1fec6b2bb5 100644 --- a/tests/data/test_csv_dataset.py +++ b/tests/data/test_csv_dataset.py @@ -184,10 +184,7 @@ def prepare_csv_file(data, filepath): df_subset = df.iloc[[1, 3, 4]] dataset = CSVDataset(src=df_subset, col_groups={"ehr": [f"ehr_{i}" for i in range(3)]}) self.assertEqual(len(dataset), 3) - np.testing.assert_allclose( - [round(i, 4) for i in dataset[1]["ehr"]], - [3.3333, 3.2353, 3.4000], - ) + np.testing.assert_allclose([round(i, 4) for i in dataset[1]["ehr"]], [3.3333, 3.2353, 3.4000]) # test pre-loaded multiple DataFrames, join tables with kwargs dfs = [pd.read_csv(i) for i in filepaths]