Repository: feature-engine/feature_engine

Issue:
drop_duplicate_features with huge datasets
When running drop_duplicate_features() with a huge dataset (More that 60K features) it lasts forever (36 hours and couting), while simple code:

       # Remove duplicated features
        duplicated_feat = []
        for i in range(0, len(X_train.columns)):
            # if i % 10 == 0:  # this helps me understand how the loop is going
            #     print(i)
 
            col_1 = X_train.columns[i]

            for col_2 in X_train.columns[i + 1:]:
                if X_train[col_1].equals(X_train[col_2]):
                    duplicated_feat.append(col_2)

Took half an hour to finish. Please check.


Comments:

Patch:
diff --git a/feature_engine/selection/drop_duplicate_features.py b/feature_engine/selection/drop_duplicate_features.py
index e4c72dc..5185384 100644
--- a/feature_engine/selection/drop_duplicate_features.py
+++ b/feature_engine/selection/drop_duplicate_features.py
@@ -1,3 +1,4 @@
+from collections import defaultdict
 from typing import List, Union
 
 import pandas as pd
@@ -99,7 +100,6 @@ class DropDuplicateFeatures(BaseSelector):
         missing_values: str = "ignore",
         confirm_variables: bool = False,
     ):
-
         if missing_values not in ["raise", "ignore"]:
             raise ValueError("missing_values takes only values 'raise' or 'ignore'.")
 
@@ -136,42 +136,30 @@ class DropDuplicateFeatures(BaseSelector):
             # check if dataset contains na
             _check_contains_na(X, self.variables_)
 
-        # create tuples of duplicated feature groups
-        self.duplicated_feature_sets_ = []
-
-        # set to collect features that are duplicated
-        self.features_to_drop_ = set()  # type: ignore
-
-        # create set of examined features
-        _examined_features = set()
-
-        for feature in self.variables_:
+        # collect duplicate features
+        _features_hashmap = defaultdict(list)
 
-            # append so we can remove when we create the combinations
-            _examined_features.add(feature)
+        # hash the features
+        _X_hash = pd.util.hash_pandas_object(X[self.variables_].T, index=False)
 
-            if feature not in self.features_to_drop_:
+        # group the features by hash
+        for feature, feature_hash in _X_hash.items():
+            _features_hashmap[feature_hash].append(feature)
 
-                _temp_set = set([feature])
-
-                # features that have not been examined, are not currently examined and
-                # were not found duplicates
-                _features_to_compare = [
-                    f
-                    for f in self.variables_
-                    if f not in _examined_features.union(self.features_to_drop_)
-                ]
-
-                # create combinations:
-                for f2 in _features_to_compare:
-
-                    if X[feature].equals(X[f2]):
-                        self.features_to_drop_.add(f2)
-                        _temp_set.add(f2)
+        # create tuples of duplicated feature groups
+        self.duplicated_feature_sets_ = [
+            set(duplicate)
+            for duplicate in _features_hashmap.values()
+            if len(duplicate) > 1
+        ]
 
-                # if there are duplicated features
-                if len(_temp_set) > 1:
-                    self.duplicated_feature_sets_.append(_temp_set)
+        # set to collect features that are duplicated
+        self.features_to_drop_ = {
+            item
+            for duplicates in _features_hashmap.values()
+            for item in duplicates[1:]
+            if duplicates and len(duplicates) > 1
+        }
 
         # save input features
         self._get_feature_names_in(X)


Test patch:
diff --git a/tests/test_selection/test_drop_duplicate_features.py b/tests/test_selection/test_drop_duplicate_features.py
index d461f50..5d33b3d 100644
--- a/tests/test_selection/test_drop_duplicate_features.py
+++ b/tests/test_selection/test_drop_duplicate_features.py
@@ -43,6 +43,23 @@ def df_duplicate_features_with_na():
     return df
 
 
+@pytest.fixture(scope="module")
+def df_duplicate_features_with_different_data_types():
+    data = {
+        "A": pd.Series([5.5] * 3).astype("float64"),
+        "B": 1,
+        "C": "foo",
+        "D": pd.Timestamp("20010102"),
+        "E": pd.Series([1.0] * 3).astype("float32"),
+        "F": False,
+        "G": pd.Series([1] * 3, dtype="int8"),
+    }
+
+    df = pd.DataFrame(data)
+
+    return df
+
+
 def test_drop_duplicates_features(df_duplicate_features):
     transformer = DropDuplicateFeatures()
     X = transformer.fit_transform(df_duplicate_features)
@@ -94,3 +111,18 @@ def test_with_df_with_na(df_duplicate_features_with_na):
         {"City", "City2"},
         {"Age", "Age2"},
     ]
+
+
+def test_with_different_data_types(df_duplicate_features_with_different_data_types):
+    transformer = DropDuplicateFeatures()
+    X = transformer.fit_transform(df_duplicate_features_with_different_data_types)
+    df = pd.DataFrame(
+        {
+            "A": pd.Series([5.5] * 3).astype("float64"),
+            "B": 1,
+            "C": "foo",
+            "D": pd.Timestamp("20010102"),
+            "F": False,
+        }
+    )
+    pd.testing.assert_frame_equal(X, df)