feat: adding remove_duplicates transformation

cristian-rincon · Nov 22, 2023 · 732fc25 · 732fc25
1 parent 2e5fa4c
commit 732fc25
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/extractor/core.py b/extractor/core.py
@@ -104,6 +104,7 @@ def extract_data(source_path: Path, format: str) -> None:
     output = pd.DataFrame(pkgs_raw_metadata)
     output["uppercased_name"] = output["name"].str.upper()
     output = output.sort_values(by=["uppercased_name"])
+    output = output.drop_duplicates(subset=["uppercased_name"], keep="first")
     del output["uppercased_name"]
     return output
 

diff --git a/tests/mocks/in/pip_freeze/sample_2.txt b/tests/mocks/in/pip_freeze/sample_2.txt
@@ -2,6 +2,7 @@ setuptools==68.0.0
 wheel==0.40.0
 dbx==0.8.10
 typing_extensions==4.5.0
+pandas
 pytest==7.3.1
 pytest-cov==4.1.0
 pre-commit==3.3.2