Repository: cytomining/pycytominer

Issue:
Questions about drop_outlier_features
I was looking at some code @fefossa is writing, and came across `drop_outlier_features`, which was added in #62 but somehow escaped my notice (it isn't on in the profiling template, for example). This is potentially super useful to us, but I was wondering about the default value (15). I know I'm asking about decisions made in December 2019, a far simpler time, but do you recall how this value was chosen? It doesn't appear at first pass to be documented, and this seems to me to be WAY too low - we routinely see features we believe in above that absolute value. Fernanda is going to check on some profiles with and without that feature reduction step added to get a sense of in her data how much effect it is having, but it is possibly significant.

I know the parameter is configurable, so we can override it if we'd like, but in general I assume we want defaults to be broadly applicable for the vast set of use cases, and I don't know that this one is. 

Discussion:

Patch:
diff --git a/pycytominer/cyto_utils/features.py b/pycytominer/cyto_utils/features.py
index 8cd55a5..8cfc43c 100644
--- a/pycytominer/cyto_utils/features.py
+++ b/pycytominer/cyto_utils/features.py
@@ -142,7 +142,7 @@ def count_na_features(population_df, features):
 
 
 def drop_outlier_features(
-    population_df, features="infer", samples="all", outlier_cutoff=15
+    population_df, features="infer", samples="all", outlier_cutoff=500
 ):
     """Exclude a feature if its min or max absolute value is greater than the threshold.
 
@@ -154,7 +154,8 @@ def drop_outlier_features(
         Features present in the population dataframe. If "infer", then assume Cell Painting features are those that start with "Cells_", "Nuclei_", or "Cytoplasm_"
     samples : list of str or str, default "all"
         Samples to perform the operation on
-    outlier_cutoff : int or float, default 15
+    outlier_cutoff : int or float, default 500
+    see https://github.com/cytomining/pycytominer/issues/237 for details.
         Threshold to remove features if absolute values is greater
 
     Returns
diff --git a/pycytominer/feature_select.py b/pycytominer/feature_select.py
index 1e70f75..c975a51 100644
--- a/pycytominer/feature_select.py
+++ b/pycytominer/feature_select.py
@@ -35,7 +35,7 @@ def feature_select(
     compression_options=None,
     float_format=None,
     blocklist_file=None,
-    outlier_cutoff=15,
+    outlier_cutoff=500,
     noise_removal_perturb_groups=None,
     noise_removal_stdev_cutoff=None,
 ):
@@ -84,8 +84,8 @@ def feature_select(
         decimal precision.
     blocklist_file : str, optional
         File location of datafrmame with with features to exclude. Note that if "blocklist" in operation then will remove standard blocklist
-    outlier_cutoff : float, default 15
-        The threshold at which the maximum or minimum value of a feature across a full experiment is excluded. Note that this procedure is typically applied (and therefore the default is uitable) for after normalization.
+    outlier_cutoff : float, default 500
+        The threshold at which the maximum or minimum value of a feature across a full experiment is excluded. Note that this procedure is typically applied after normalization.
     noise_removal_perturb_groups: str or list of str, optional
         Perturbation groups corresponding to rows in profiles or the the name of the metadata column containing this information.
     noise_removal_stdev_cutoff: float,optional


Test patch:
diff --git a/pycytominer/tests/test_cyto_utils/test_feature_drop_outlier.py b/pycytominer/tests/test_cyto_utils/test_feature_drop_outlier.py
index ab190de..303b18b 100644
--- a/pycytominer/tests/test_cyto_utils/test_feature_drop_outlier.py
+++ b/pycytominer/tests/test_cyto_utils/test_feature_drop_outlier.py
@@ -23,29 +23,29 @@ data_df = pd.DataFrame(
         "Cells_x": [1, 2, -8, 2, 5, 5, 5, -1],
         "Cytoplasm_y": [3, -1, 7, 4, 5, -9, 6, 1],
         "Nuclei_z": [-1, 8, 2, 5, -6, 20, 2, -2],
-        "Cells_zz": [14, -46, 1, 60, -30, -100, 2, 2],
+        "Cells_zz": [14, -46, 1, 60, -30, -10000, 2, 2],
     }
 ).reset_index(drop=True)
 
 
 def test_outlier_default():
     result = drop_outlier_features(data_df)
-    expected_result = ["Cells_zz", "Nuclei_z"]
+    expected_result = ["Cells_zz"]
     assert sorted(result) == sorted(expected_result)
 
 
-def test_outlier_high_cutoff():
-    result = drop_outlier_features(data_df, outlier_cutoff=30)
-    expected_result = ["Cells_zz"]
-    assert result == expected_result
+def test_outlier_15_cutoff():
+    result = drop_outlier_features(data_df, outlier_cutoff=15)
+    expected_result = ["Cells_zz", "Nuclei_z"]
+    assert sorted(result) == sorted(expected_result)
 
 
-def test_outlier_samples():
-    result = drop_outlier_features(data_df, samples=[0, 1, 2, 3, 5])
+def test_outlier_samples_15():
+    result = drop_outlier_features(data_df, samples=[0, 1, 2, 3, 5], outlier_cutoff=15)
     expected_result = ["Cells_zz", "Nuclei_z"]
     assert sorted(result) == sorted(expected_result)
 
-    result = drop_outlier_features(data_df, samples=[0, 1, 2, 3])
+    result = drop_outlier_features(data_df, samples=[0, 1, 2, 3], outlier_cutoff=15)
     expected_result = ["Cells_zz"]
     assert result == expected_result
 
diff --git a/pycytominer/tests/test_feature_select.py b/pycytominer/tests/test_feature_select.py
index f498fdd..5474729 100644
--- a/pycytominer/tests/test_feature_select.py
+++ b/pycytominer/tests/test_feature_select.py
@@ -67,7 +67,7 @@ data_outlier_df = pd.DataFrame(
         "Cells_x": [1, 2, -8, 2, 5, 5, 5, -1],
         "Cytoplasm_y": [3, -1, 7, 4, 5, -9, 6, 1],
         "Nuclei_z": [-1, 8, 2, 5, -6, 20, 2, -2],
-        "Cells_zz": [14, -46, 1, 60, -30, -100, 2, 2],
+        "Cells_zz": [14, -46, 1, 60, -30, -10000, 2, 2],
     }
 ).reset_index(drop=True)
 
@@ -425,7 +425,7 @@ def test_feature_select_drop_outlier():
     result = feature_select(
         data_outlier_df, features="infer", operation="drop_outliers"
     )
-    expected_result = data_outlier_df.drop(["Cells_zz", "Nuclei_z"], axis="columns")
+    expected_result = data_outlier_df.drop(["Cells_zz"], axis="columns")
     pd.testing.assert_frame_equal(result, expected_result)
 
     result = feature_select(
@@ -435,6 +435,6 @@ def test_feature_select_drop_outlier():
     pd.testing.assert_frame_equal(result, expected_result)
 
     result = feature_select(
-        data_outlier_df, features=["Cells_x", "Cytoplasm_y"], operation="drop_outliers"
+        data_outlier_df, features=["Cells_x", "Cytoplasm_y"], operation="drop_outliers", outlier_cutoff=15
     )
     pd.testing.assert_frame_equal(result, data_outlier_df)