Repository: blue-yonder/tsfresh

Issue:
Add feature class: Count/Percentage same value
I had an idea for a class of features. 

1.  percentage of data points that occur at least second time 
2. sum of data points that occur at least second time 
3. percentage of observed values that occur at lest second time 
...

Discussion:
coveralls: 
[![Coverage Status](https://coveralls.io/builds/8863140/badge)](https://coveralls.io/builds/8863140)

Coverage increased (+0.2%) to 93.674% when pulling **bd41d0dab7dffa94cdc7614bb8d311b16c1cc4bf on nils-braun:count-percentage-of-some-value** into **50b3ee7ee663aacf8d6ae28bcb2bfcc00bd3d9a1 on blue-yonder:master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/8863140/badge)](https://coveralls.io/builds/8863140)

Coverage increased (+0.2%) to 93.674% when pulling **bd41d0dab7dffa94cdc7614bb8d311b16c1cc4bf on nils-braun:count-percentage-of-some-value** into **50b3ee7ee663aacf8d6ae28bcb2bfcc00bd3d9a1 on blue-yonder:master**.

MaxBenChrist: Awesome @nils-braun !


Patch:
diff --git a/tsfresh/feature_extraction/feature_calculators.py b/tsfresh/feature_extraction/feature_calculators.py
index 5882882..d216254 100644
--- a/tsfresh/feature_extraction/feature_calculators.py
+++ b/tsfresh/feature_extraction/feature_calculators.py
@@ -587,6 +587,92 @@ def first_location_of_minimum(x):
     return np.argmin(x) / len(x) if len(x) > 0 else np.NaN
 
 
+@set_property("fctype", "aggregate")
+@not_apply_to_raw_numbers
+def percentage_of_reoccurring_datapoints_to_all_datapoints(x):
+    """
+    Returns the percentage of unique values, that are present in the time series
+    more than once.
+
+        len(different values occurring more than once) / len(different values)
+
+    This means the percentage is normalized to the number of unique values,
+    in contrast to the percentage_of_reoccurring_values_to_all_values.
+
+    :param x: the time series to calculate the feature of
+    :type x: pandas.Series
+    :return: the value of this feature
+    :return type: float
+    """
+    x = pd.Series(x)
+    return (x.value_counts() > 1).mean()
+
+
+@set_property("fctype", "aggregate")
+@not_apply_to_raw_numbers
+def percentage_of_reoccurring_values_to_all_values(x):
+    """
+    Returns the ratio of unique values, that are present in the time series
+    more than once.
+
+        # of data points occurring more than once / # of all data points
+
+    This means the ratio is normalized to the number of data points in the time series,
+    in contrast to the percentage_of_reoccurring_datapoints_to_all_datapoints.
+
+    :param x: the time series to calculate the feature of
+    :type x: pandas.Series
+    :return: the value of this feature
+    :return type: float
+    """
+    x = pd.Series(x)
+
+    if len(x) == 0:
+        return np.nan
+
+    value_counts = x.value_counts()
+    return 1.0 * value_counts[value_counts > 1].sum() / len(x)
+
+
+@set_property("fctype", "aggregate")
+@not_apply_to_raw_numbers
+def sum_of_reoccurring_values(x):
+    """
+    Returns the sum of all values, that are present in the time series
+    more than once.
+
+    :param x: the time series to calculate the feature of
+    :type x: pandas.Series
+    :return: the value of this feature
+    :return type: float
+    """
+    x = pd.Series(x)
+    value_counts = x.value_counts()
+    doubled_values = value_counts[value_counts > 1]
+    return sum(doubled_values.index * doubled_values)
+
+
+@set_property("fctype", "aggregate")
+@not_apply_to_raw_numbers
+def ratio_value_number_to_time_series_length(x):
+    """
+    Returns a factor which is 1 if all values in the time series occur only once, and below one if this is not the
+    case. In principle, it just returns
+
+        # unique values / # values
+
+    :param x: the time series to calculate the feature of
+    :type x: pandas.Series
+    :return: the value of this feature
+    :return type: float
+    """
+
+    if len(x) == 0:
+        return np.nan
+
+    return 1.0 * len(set(x))/len(x)
+
+
 @set_property("fctype", "apply")
 @not_apply_to_raw_numbers
 def fft_coefficient(x, c, param):
@@ -723,12 +809,12 @@ def cwt_coefficients(x, c, param):
 
     for widths in df_cfg["widths"].unique():
 
-        coeff = df_cfg[df_cfg["widths"] == widths]["coeff"].unique()
         # the calculated_cwt will shape (len(widths), len(x)).
         calculated_cwt = cwt(x, ricker, widths)
 
         for w in df_cfg[df_cfg["widths"] == widths]["w"].unique():
 
+            coeff = df_cfg[(df_cfg["widths"] == widths) & (df_cfg["w"] == w)]["coeff"].unique()
             i = widths.index(w)
 
             if calculated_cwt.shape[1] < len(coeff):  # There are less data points than requested model coefficients


Test patch:
diff --git a/tests/feature_extraction/test_feature_calculations.py b/tests/feature_extraction/test_feature_calculations.py
index e33c376..32cb43f 100644
--- a/tests/feature_extraction/test_feature_calculations.py
+++ b/tests/feature_extraction/test_feature_calculations.py
@@ -258,6 +258,34 @@ class FeatureCalculationTestCase(TestCase):
         self.assertAlmostEqualOnAllArrayTypes(first_location_of_minimum, [1], 0.0)
         self.assertIsNanOnAllArrayTypes(first_location_of_minimum, [])
 
+    def test_percentage_of_doubled_datapoints(self):
+        self.assertAlmostEqualOnAllArrayTypes(percentage_of_reoccurring_datapoints_to_all_datapoints, [1, 1, 2, 3, 4], 0.25)
+        self.assertAlmostEqualOnAllArrayTypes(percentage_of_reoccurring_datapoints_to_all_datapoints, [1, 1.5, 2, 3], 0)
+        self.assertAlmostEqualOnAllArrayTypes(percentage_of_reoccurring_datapoints_to_all_datapoints, [1], 0)
+        self.assertAlmostEqualOnAllArrayTypes(percentage_of_reoccurring_datapoints_to_all_datapoints, [1.111, -2.45, 1.111, 2.45], 1.0 / 3.0)
+        self.assertIsNanOnAllArrayTypes(percentage_of_reoccurring_datapoints_to_all_datapoints, [])
+
+    def test_ratio_of_doubled_values(self):
+        self.assertAlmostEqualOnAllArrayTypes(percentage_of_reoccurring_values_to_all_values, [1, 1, 2, 3, 4], 0.4)
+        self.assertAlmostEqualOnAllArrayTypes(percentage_of_reoccurring_values_to_all_values, [1, 1.5, 2, 3], 0)
+        self.assertAlmostEqualOnAllArrayTypes(percentage_of_reoccurring_values_to_all_values, [1], 0)
+        self.assertAlmostEqualOnAllArrayTypes(percentage_of_reoccurring_values_to_all_values, [1.111, -2.45, 1.111, 2.45], 0.5)
+        self.assertIsNanOnAllArrayTypes(percentage_of_reoccurring_values_to_all_values, [])
+
+    def test_sum_of_doubled_values(self):
+        self.assertAlmostEqualOnAllArrayTypes(sum_of_reoccurring_values, [1, 1, 2, 3, 4], 2)
+        self.assertAlmostEqualOnAllArrayTypes(sum_of_reoccurring_values, [1, 1.5, 2, 3], 0)
+        self.assertAlmostEqualOnAllArrayTypes(sum_of_reoccurring_values, [1], 0)
+        self.assertAlmostEqualOnAllArrayTypes(sum_of_reoccurring_values, [1.111, -2.45, 1.111, 2.45], 2.222)
+        self.assertAlmostEqualOnAllArrayTypes(sum_of_reoccurring_values, [], 0)
+
+    def test_uniqueness_factor(self):
+        self.assertAlmostEqualOnAllArrayTypes(ratio_value_number_to_time_series_length, [1, 1, 2, 3, 4], 0.8)
+        self.assertAlmostEqualOnAllArrayTypes(ratio_value_number_to_time_series_length, [1, 1.5, 2, 3], 1)
+        self.assertAlmostEqualOnAllArrayTypes(ratio_value_number_to_time_series_length, [1], 1)
+        self.assertAlmostEqualOnAllArrayTypes(ratio_value_number_to_time_series_length, [1.111, -2.45, 1.111, 2.45], 0.75)
+        self.assertIsNanOnAllArrayTypes(ratio_value_number_to_time_series_length, [])
+
     def test_fft_coefficient(self):
         pass
         # todo: add unit test