Merge pull request #142 from aakankshaduggal/drop-index-0-columns

Drop `__index_level_0__` columns
instructlab · Jul 16, 2024 · 5c52d0f · 5c52d0f
2 parents ac409fc + a3681ee
commit 5c52d0f
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/src/instructlab/sdg/pipeline.py b/src/instructlab/sdg/pipeline.py
@@ -57,8 +57,9 @@ def _drop_duplicates(self, dataset, cols):
         Drop duplicates from the dataset based on the columns provided.
         """
         df = dataset.to_pandas()
-        df.drop_duplicates(subset=cols, inplace=True)
-        return Dataset.from_pandas(df)
+        df = df.drop_duplicates(subset=cols).reset_index(drop=True)
+        ds = Dataset.from_pandas(df)
+        return ds
 
     def generate(self, dataset) -> Dataset:
         """