Repository: blue-yonder/tsfresh

Issue:
Unable to disable warnings in extract_features
I am unable to disable the warning messages.  I have tried all of the following:

```
import warnings
warnings.simplefilter("ignore")
warnings.filterwarnings("ignore")
warnings.filterwarnings("ignore", category=RuntimeWarning)
numpy.warnings.filterwarnings('ignore')
```

I have also tried the switch:

```
X = extract_features(df_roll, column_id="id", column_sort="time", column_value="value",
                         impute_function=impute, disable_progressbar=True, show_warnings=False)
```

I have also tried suppressing stderr and stdout...

Is there any way to do it?  I have many columns to calculate and I get many RuntimeWarnings and RankWarning.  It floods my console and is redundant for each column.  Please advise.

1. Your operating system
Windows 10
2. The version of *tsfresh* that you are using
Latest
3. The data on which the problem occurred (please do not upload 1000s of time series but try to boil the problem down to a small group or even a singular one)
All time series
4. A minimal code snippet which reproduces the problem/bug
X = extract_features(df_roll, column_id="id", column_sort="time", column_value="value",
                         impute_function=impute, disable_progressbar=True, show_warnings=False)
5. Any reported errors or traceback

For questions, you can also use our [gitter chatroom](https://gitter.im/tsfresh/)


Discussion:
coveralls: 
[![Coverage Status](https://coveralls.io/builds/27288268/badge)](https://coveralls.io/builds/27288268)

Coverage decreased (-0.06%) to 97.396% when pulling **4ea04e57dfb6130b7e46d0845bc10a5364340380 on dbarbier:db/show_warnings** into **ceb6bd0c3e437a44394098cda4225276f9316681 on blue-yonder:master**.

nils-braun: As always, thank you very much for your great work!

I am fine with the changes as they are. If no one else wants to comment I will merge it soon(ish). Sorry for the delay.

Patch:
diff --git a/tsfresh/convenience/relevant_extraction.py b/tsfresh/convenience/relevant_extraction.py
index 36ed1ca..82c83da 100644
--- a/tsfresh/convenience/relevant_extraction.py
+++ b/tsfresh/convenience/relevant_extraction.py
@@ -87,7 +87,7 @@ def extract_relevant_features(timeseries_container, y, X=None,
     :param n_jobs: The number of processes to use for parallelization. If zero, no parallelization is used.
     :type n_jobs: int
 
-    :param: show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
+    :param show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
     :type show_warnings: bool
 
     :param disable_progressbar: Do not show a progressbar while doing the calculation.
@@ -170,6 +170,7 @@ def extract_relevant_features(timeseries_container, y, X=None,
                             test_for_real_target_real_feature=test_for_real_target_real_feature,
                             fdr_level=fdr_level, hypotheses_independent=hypotheses_independent,
                             n_jobs=n_jobs,
+                            show_warnings=show_warnings,
                             chunksize=chunksize,
                             ml_task=ml_task)
 
diff --git a/tsfresh/feature_extraction/extraction.py b/tsfresh/feature_extraction/extraction.py
index d66cf62..15d2f91 100644
--- a/tsfresh/feature_extraction/extraction.py
+++ b/tsfresh/feature_extraction/extraction.py
@@ -106,7 +106,7 @@ def extract_features(timeseries_container, default_fc_parameters=None,
         smaller chunksize.
     :type chunksize: None or int
 
-    :param: show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
+    :param show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
     :type show_warnings: bool
 
     :param disable_progressbar: Do not show a progressbar while doing the calculation.
@@ -163,6 +163,7 @@ def extract_features(timeseries_container, default_fc_parameters=None,
                                 column_kind=column_kind,
                                 n_jobs=n_jobs, chunk_size=chunksize,
                                 disable_progressbar=disable_progressbar,
+                                show_warnings=show_warnings,
                                 default_fc_parameters=default_fc_parameters,
                                 kind_to_fc_parameters=kind_to_fc_parameters,
                                 distributor=distributor)
@@ -243,7 +244,7 @@ def generate_data_chunk_format(df, column_id, column_kind, column_value):
 
 def _do_extraction(df, column_id, column_value, column_kind,
                    default_fc_parameters, kind_to_fc_parameters,
-                   n_jobs, chunk_size, disable_progressbar, distributor):
+                   n_jobs, chunk_size, disable_progressbar, show_warnings, distributor):
     """
     Wrapper around the _do_extraction_on_chunk, which calls it on all chunks in the data frame.
     A chunk is a subset of the data, with a given kind and id - so a single time series.
@@ -285,6 +286,9 @@ def _do_extraction(df, column_id, column_value, column_kind,
     :param disable_progressbar: Do not show a progressbar while doing the calculation.
     :type disable_progressbar: bool
 
+    :param show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
+    :type show_warnings: bool
+
     :param distributor: Advanced parameter:  See the utilities/distribution.py for more information.
                          Leave to None, if you want TSFresh to choose the best distributor.
     :type distributor: DistributorBaseClass
@@ -302,7 +306,8 @@ def _do_extraction(df, column_id, column_value, column_kind,
         else:
             distributor = MultiprocessingDistributor(n_workers=n_jobs,
                                                      disable_progressbar=disable_progressbar,
-                                                     progressbar_title="Feature Extraction")
+                                                     progressbar_title="Feature Extraction",
+                                                     show_warnings=show_warnings)
 
     if not isinstance(distributor, DistributorBaseClass):
         raise ValueError("the passed distributor is not an DistributorBaseClass object")
@@ -317,7 +322,7 @@ def _do_extraction(df, column_id, column_value, column_kind,
 
     # Return a dataframe in the typical form (id as index and feature names as columns)
     result = pd.DataFrame(result)
-    if result.columns.contains("value"):
+    if "value" in result.columns:
         result["value"] = result["value"].astype(float)
 
     if len(result) != 0:
diff --git a/tsfresh/feature_extraction/feature_calculators.py b/tsfresh/feature_extraction/feature_calculators.py
index e1b8cda..be766b5 100644
--- a/tsfresh/feature_extraction/feature_calculators.py
+++ b/tsfresh/feature_extraction/feature_calculators.py
@@ -15,6 +15,7 @@ seen by tsfresh as a feature calculator. Others will not be calculated.
 """
 
 import itertools
+import functools
 import warnings
 from builtins import range
 from collections import defaultdict
@@ -427,26 +428,41 @@ def augmented_dickey_fuller(x, param):
 
     :param x: the time series to calculate the feature of
     :type x: numpy.ndarray
-    :param param: contains dictionaries {"attr": x} with x str, either "teststat", "pvalue" or "usedlag"
+    :param param: contains dictionaries {"attr": x, "autolag": y} with x str, either "teststat", "pvalue" or "usedlag"
+                  and with y str, either of "AIC", "BIC", "t-stats" or None (See the documentation of adfuller() in
+                  statsmodels).
     :type param: list
     :return: the value of this feature
     :return type: float
     """
-    res = None
-    try:
-        res = adfuller(x)
-    except LinAlgError:
-        res = np.NaN, np.NaN, np.NaN
-    except ValueError:  # occurs if sample size is too small
-        res = np.NaN, np.NaN, np.NaN
-    except MissingDataError:  # is thrown for e.g. inf or nan in the data
-        res = np.NaN, np.NaN, np.NaN
-
-    return [('attr_"{}"'.format(config["attr"]),
-             res[0] if config["attr"] == "teststat"
-             else res[1] if config["attr"] == "pvalue"
-             else res[2] if config["attr"] == "usedlag" else np.NaN)
-            for config in param]
+
+    @functools.lru_cache()
+    def compute_adf(autolag):
+        try:
+            return adfuller(x, autolag=autolag)
+        except LinAlgError:
+            return np.NaN, np.NaN, np.NaN
+        except ValueError:  # occurs if sample size is too small
+            return np.NaN, np.NaN, np.NaN
+        except MissingDataError:  # is thrown for e.g. inf or nan in the data
+            return np.NaN, np.NaN, np.NaN
+
+    res = []
+    for config in param:
+        autolag = config.get("autolag", "AIC")
+
+        adf = compute_adf(autolag)
+        index = 'autolag_"{}"__attr_"{}"'.format(autolag, config["attr"])
+
+        if config["attr"] == "teststat":
+            res.append((index, adf[0]))
+        elif config["attr"] == "pvalue":
+            res.append((index, adf[1]))
+        elif config["attr"] == "usedlag":
+            res.append((index, adf[2]))
+        else:
+            res.append((index, np.NaN))
+    return res
 
 
 @set_property("fctype", "simple")
@@ -1461,6 +1477,7 @@ def binned_entropy(x, max_bins):
     probs[probs == 0] = 1.0
     return - np.sum(probs * np.log(probs))
 
+
 # todo - include latex formula
 # todo - check if vectorizable
 @set_property("high_comp_cost", True)
diff --git a/tsfresh/feature_selection/relevance.py b/tsfresh/feature_selection/relevance.py
index add520c..f73642f 100644
--- a/tsfresh/feature_selection/relevance.py
+++ b/tsfresh/feature_selection/relevance.py
@@ -10,11 +10,10 @@ Afterwards the Benjamini Hochberg procedure which is a multiple testing procedur
 which to cut off (solely based on the p-values).
 """
 
-import logging
 from multiprocessing import Pool
+import warnings
 
 import numpy as np
-import os
 import pandas as pd
 from functools import partial, reduce
 
@@ -22,11 +21,11 @@ from tsfresh import defaults
 from tsfresh.feature_selection.benjamini_hochberg_test import benjamini_hochberg_test
 from tsfresh.feature_selection.significance_tests import target_binary_feature_real_test, \
     target_real_feature_binary_test, target_real_feature_real_test, target_binary_feature_binary_test
+from tsfresh.utilities.distribution import initialize_warnings_in_workers
 
-_logger = logging.getLogger(__name__)
 
-
-def calculate_relevance_table(X, y, ml_task='auto', n_jobs=defaults.N_PROCESSES, chunksize=defaults.CHUNKSIZE,
+def calculate_relevance_table(X, y, ml_task='auto', n_jobs=defaults.N_PROCESSES,
+                              show_warnings=defaults.SHOW_WARNINGS, chunksize=defaults.CHUNKSIZE,
                               test_for_binary_target_binary_feature=defaults.TEST_FOR_BINARY_TARGET_BINARY_FEATURE,
                               test_for_binary_target_real_feature=defaults.TEST_FOR_BINARY_TARGET_REAL_FEATURE,
                               test_for_real_target_binary_feature=defaults.TEST_FOR_REAL_TARGET_BINARY_FEATURE,
@@ -110,6 +109,9 @@ def calculate_relevance_table(X, y, ml_task='auto', n_jobs=defaults.N_PROCESSES,
     :param n_jobs: Number of processes to use during the p-value calculation
     :type n_jobs: int
 
+    :param show_warnings: Show warnings during the p-value calculation (needed for debugging of calculators).
+    :type show_warnings: bool
+
     :param chunksize: The size of one chunk that is submitted to the worker
         process for the parallelisation.  Where one chunk is defined as a
         singular time series for one id and one kind. If you set the chunksize
@@ -134,59 +136,74 @@ def calculate_relevance_table(X, y, ml_task='auto', n_jobs=defaults.N_PROCESSES,
     elif ml_task == 'auto':
         ml_task = infer_ml_task(y)
 
-    if n_jobs == 0:
-        map_function = map
-    else:
-        pool = Pool(n_jobs)
-        map_function = partial(pool.map, chunksize=chunksize)
-
-    relevance_table = pd.DataFrame(index=pd.Series(X.columns, name='feature'))
-    relevance_table['feature'] = relevance_table.index
-    relevance_table['type'] = pd.Series(
-        map_function(get_feature_type, [X[feature] for feature in relevance_table.index]),
-        index=relevance_table.index
-    )
-    table_real = relevance_table[relevance_table.type == 'real'].copy()
-    table_binary = relevance_table[relevance_table.type == 'binary'].copy()
-
-    table_const = relevance_table[relevance_table.type == 'constant'].copy()
-    table_const['p_value'] = np.NaN
-    table_const['relevant'] = False
-
-    if len(table_const) == len(relevance_table):
-        return table_const
-
-    if ml_task == 'classification':
-        tables = []
-        for label in y.unique():
-            _test_real_feature = partial(target_binary_feature_real_test, y=(y == label),
-                                         test=test_for_binary_target_real_feature)
-            _test_binary_feature = partial(target_binary_feature_binary_test, y=(y == label))
-            tmp = _calculate_relevance_table_for_implicit_target(
+    with warnings.catch_warnings():
+        if not show_warnings:
+            warnings.simplefilter("ignore")
+        else:
+            warnings.simplefilter("default")
+
+        if n_jobs == 0:
+            map_function = map
+        else:
+            pool = Pool(processes=n_jobs, initializer=initialize_warnings_in_workers, initargs=(show_warnings,))
+            map_function = partial(pool.map, chunksize=chunksize)
+
+        relevance_table = pd.DataFrame(index=pd.Series(X.columns, name='feature'))
+        relevance_table['feature'] = relevance_table.index
+        relevance_table['type'] = pd.Series(
+            map_function(get_feature_type, [X[feature] for feature in relevance_table.index]),
+            index=relevance_table.index
+        )
+        table_real = relevance_table[relevance_table.type == 'real'].copy()
+        table_binary = relevance_table[relevance_table.type == 'binary'].copy()
+
+        table_const = relevance_table[relevance_table.type == 'constant'].copy()
+        table_const['p_value'] = np.NaN
+        table_const['relevant'] = False
+
+        if not table_const.empty:
+            warnings.warn("[test_feature_significance] Constant features: {}"
+                          .format(", ".join(table_const.feature)), RuntimeWarning)
+
+        if len(table_const) == len(relevance_table):
+            if n_jobs != 0:
+                pool.close()
+                pool.terminate()
+                pool.join()
+            return table_const
+
+        if ml_task == 'classification':
+            tables = []
+            for label in y.unique():
+                _test_real_feature = partial(target_binary_feature_real_test, y=(y == label),
+                                             test=test_for_binary_target_real_feature)
+                _test_binary_feature = partial(target_binary_feature_binary_test, y=(y == label))
+                tmp = _calculate_relevance_table_for_implicit_target(
+                    table_real, table_binary, X, _test_real_feature, _test_binary_feature, hypotheses_independent,
+                    fdr_level, map_function
+                )
+                tables.append(tmp)
+            relevance_table = combine_relevance_tables(tables)
+        elif ml_task == 'regression':
+            _test_real_feature = partial(target_real_feature_real_test, y=y)
+            _test_binary_feature = partial(target_real_feature_binary_test, y=y)
+            relevance_table = _calculate_relevance_table_for_implicit_target(
                 table_real, table_binary, X, _test_real_feature, _test_binary_feature, hypotheses_independent,
                 fdr_level, map_function
             )
-            tables.append(tmp)
-        relevance_table = combine_relevance_tables(tables)
-    elif ml_task == 'regression':
-        _test_real_feature = partial(target_real_feature_real_test, y=y)
-        _test_binary_feature = partial(target_real_feature_binary_test, y=y)
-        relevance_table = _calculate_relevance_table_for_implicit_target(
-            table_real, table_binary, X, _test_real_feature, _test_binary_feature, hypotheses_independent, fdr_level,
-            map_function
-        )
 
-    relevance_table = pd.concat([relevance_table, table_const], axis=0)
+        if n_jobs != 0:
+            pool.close()
+            pool.terminate()
+            pool.join()
 
-    if n_jobs != 0:
-        pool.close()
-        pool.terminate()
-        pool.join()
+        relevance_table = pd.concat([relevance_table, table_const], axis=0)
 
-    if sum(relevance_table['relevant']) == 0:
-        _logger.warning(
-            "No feature was found relevant for {} for fdr level = {} (which corresponds to the maximal percentage of "
-            "irrelevant features, consider using an higher fdr level or add other features.".format(ml_task, fdr_level))
+        if sum(relevance_table['relevant']) == 0:
+            warnings.warn(
+                "No feature was found relevant for {} for fdr level = {} (which corresponds to the maximal percentage "
+                "of irrelevant features, consider using an higher fdr level or add other features."
+                .format(ml_task, fdr_level), RuntimeWarning)
 
     return relevance_table
 
@@ -222,7 +239,6 @@ def infer_ml_task(y):
     else:
         ml_task = 'regression'
 
-    _logger.warning('Infered {} as machine learning task'.format(ml_task))
     return ml_task
 
 
@@ -255,7 +271,6 @@ def get_feature_type(feature_column):
     """
     n_unique_values = len(set(feature_column.values))
     if n_unique_values == 1:
-        _logger.warning("[test_feature_significance] Feature {} is constant".format(feature_column.name))
         return 'constant'
     elif n_unique_values == 2:
         return 'binary'
diff --git a/tsfresh/feature_selection/selection.py b/tsfresh/feature_selection/selection.py
index 3e6c88f..e22b344 100644
--- a/tsfresh/feature_selection/selection.py
+++ b/tsfresh/feature_selection/selection.py
@@ -6,7 +6,6 @@ This module contains the filtering process for the extracted features. The filte
 other features that are not based on time series.
 """
 
-import logging
 import pandas as pd
 import numpy as np
 from tsfresh import defaults
@@ -14,16 +13,13 @@ from tsfresh.utilities.dataframe_functions import check_for_nans_in_columns
 from tsfresh.feature_selection.relevance import calculate_relevance_table
 
 
-_logger = logging.getLogger(__name__)
-
-
 def select_features(X, y, test_for_binary_target_binary_feature=defaults.TEST_FOR_BINARY_TARGET_BINARY_FEATURE,
                     test_for_binary_target_real_feature=defaults.TEST_FOR_BINARY_TARGET_REAL_FEATURE,
                     test_for_real_target_binary_feature=defaults.TEST_FOR_REAL_TARGET_BINARY_FEATURE,
                     test_for_real_target_real_feature=defaults.TEST_FOR_REAL_TARGET_REAL_FEATURE,
                     fdr_level=defaults.FDR_LEVEL, hypotheses_independent=defaults.HYPOTHESES_INDEPENDENT,
-                    n_jobs=defaults.N_PROCESSES, chunksize=defaults.CHUNKSIZE,
-                    ml_task='auto'):
+                    n_jobs=defaults.N_PROCESSES, show_warnings=defaults.SHOW_WARNINGS,
+                    chunksize=defaults.CHUNKSIZE, ml_task='auto'):
     """
     Check the significance of all features (columns) of feature matrix X and return a possibly reduced feature matrix
     only containing relevant features.
@@ -105,6 +101,9 @@ def select_features(X, y, test_for_binary_target_binary_feature=defaults.TEST_FO
     :param n_jobs: Number of processes to use during the p-value calculation
     :type n_jobs: int
 
+    :param show_warnings: Show warnings during the p-value calculation (needed for debugging of calculators).
+    :type show_warnings: bool
+
     :param chunksize: The size of one chunk that is submitted to the worker
         process for the parallelisation.  Where one chunk is defined as a
         singular time series for one id and one kind. If you set the chunksize
@@ -142,7 +141,7 @@ def select_features(X, y, test_for_binary_target_binary_feature=defaults.TEST_FO
         y = pd.Series(y, index=X.index)
 
     relevance_table = calculate_relevance_table(
-        X, y, ml_task=ml_task, n_jobs=n_jobs, chunksize=chunksize,
+        X, y, ml_task=ml_task, n_jobs=n_jobs, show_warnings=show_warnings, chunksize=chunksize,
         test_for_binary_target_real_feature=test_for_binary_target_real_feature,
         fdr_level=fdr_level, hypotheses_independent=hypotheses_independent,
     )
diff --git a/tsfresh/transformers/feature_augmenter.py b/tsfresh/transformers/feature_augmenter.py
index f40313a..218dc21 100644
--- a/tsfresh/transformers/feature_augmenter.py
+++ b/tsfresh/transformers/feature_augmenter.py
@@ -103,7 +103,7 @@ class FeatureAugmenter(BaseEstimator, TransformerMixin):
             smaller chunksize.
         :type chunksize: None or int
 
-        :param: show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
+        :param show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
         :type show_warnings: bool
 
         :param disable_progressbar: Do not show a progressbar while doing the calculation.
diff --git a/tsfresh/transformers/relevant_feature_augmenter.py b/tsfresh/transformers/relevant_feature_augmenter.py
index 6c05405..2f1bdbc 100644
--- a/tsfresh/transformers/relevant_feature_augmenter.py
+++ b/tsfresh/transformers/relevant_feature_augmenter.py
@@ -145,7 +145,7 @@ class RelevantFeatureAugmenter(BaseEstimator, TransformerMixin):
         :param n_jobs: The number of processes to use for parallelization. If zero, no parallelization is used.
         :type n_jobs: int
 
-        :param: show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
+        :param show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
         :type show_warnings: bool
 
         :param disable_progressbar: Do not show a progressbar while doing the calculation.
diff --git a/tsfresh/utilities/dataframe_functions.py b/tsfresh/utilities/dataframe_functions.py
index d60e9f9..015c7dc 100644
--- a/tsfresh/utilities/dataframe_functions.py
+++ b/tsfresh/utilities/dataframe_functions.py
@@ -10,9 +10,6 @@ import warnings
 
 import numpy as np
 import pandas as pd
-import logging
-
-_logger = logging.getLogger(__name__)
 
 
 def check_for_nans_in_columns(df, columns=None):
@@ -169,8 +166,8 @@ def get_range_values_per_column(df):
 
     if np.any(is_col_non_finite):
         # We have columns that does not contain any finite value at all, so we will store 0 instead.
-        _logger.warning("The columns {} did not have any finite values. Filling with zeros.".format(
-            df.iloc[:, np.where(is_col_non_finite)[0]].columns.values))
+        warnings.warn("The columns {} did not have any finite values. Filling with zeros.".format(
+            df.iloc[:, np.where(is_col_non_finite)[0]].columns.values), RuntimeWarning)
 
         masked.data[:, is_col_non_finite] = 0  # Set the values of the columns to 0
         masked.mask[:, is_col_non_finite] = False  # Remove the mask for this column
diff --git a/tsfresh/utilities/distribution.py b/tsfresh/utilities/distribution.py
index e13ac8b..0490b8a 100644
--- a/tsfresh/utilities/distribution.py
+++ b/tsfresh/utilities/distribution.py
@@ -10,6 +10,7 @@ Design of this module by Nils Braun
 
 import math
 import itertools
+import warnings
 from collections import Iterable
 from functools import partial
 from multiprocessing import Pool
@@ -39,6 +40,23 @@ def _function_with_partly_reduce(chunk_list, map_function, kwargs):
     return results
 
 
+def initialize_warnings_in_workers(show_warnings):
+    """
+    Small helper function to initialize warnings module in multiprocessing workers.
+
+    On Windows, Python spawns fresh processes which do not inherit from warnings
+    state, so warnings must be enabled/disabled before running computations.
+
+    :param show_warnings: whether to show warnings or not.
+    :type show_warnings: bool
+    """
+    warnings.catch_warnings()
+    if not show_warnings:
+        warnings.simplefilter("ignore")
+    else:
+        warnings.simplefilter("default")
+
+
 class DistributorBaseClass:
     """
     The distributor abstract base class.
@@ -337,7 +355,8 @@ class MultiprocessingDistributor(DistributorBaseClass):
     Distributor using a multiprocessing Pool to calculate the jobs in parallel on the local machine.
     """
 
-    def __init__(self, n_workers, disable_progressbar=False, progressbar_title="Feature Extraction"):
+    def __init__(self, n_workers, disable_progressbar=False, progressbar_title="Feature Extraction",
+                 show_warnings=True):
         """
         Creates a new MultiprocessingDistributor instance
 
@@ -347,8 +366,10 @@ class MultiprocessingDistributor(DistributorBaseClass):
         :type disable_progressbar: bool
         :param progressbar_title: the title of the progressbar
         :type progressbar_title: basestring
+        :param show_warnings: whether to show warnings or not.
+        :type show_warnings: bool
         """
-        self.pool = Pool(processes=n_workers)
+        self.pool = Pool(processes=n_workers, initializer=initialize_warnings_in_workers, initargs=(show_warnings,))
         self.n_workers = n_workers
         self.disable_progressbar = disable_progressbar
         self.progressbar_title = progressbar_title


Test patch:
diff --git a/tests/units/feature_extraction/test_feature_calculations.py b/tests/units/feature_extraction/test_feature_calculations.py
index 09860cb..8924383 100644
--- a/tests/units/feature_extraction/test_feature_calculations.py
+++ b/tests/units/feature_extraction/test_feature_calculations.py
@@ -251,14 +251,22 @@ class FeatureCalculationTestCase(TestCase):
         # H0 is true
         np.random.seed(seed=42)
         x = np.cumsum(np.random.uniform(size=100))
-        param = [{"attr": "teststat"}, {"attr": "pvalue"}, {"attr": "usedlag"}]
-        expected_index = ['attr_"teststat"', 'attr_"pvalue"', 'attr_"usedlag"']
+        param = [
+            {"autolag": "BIC", "attr": "teststat"},
+            {"autolag": "BIC", "attr": "pvalue"},
+            {"autolag": "BIC", "attr": "usedlag"}
+        ]
+        expected_index = [
+            'autolag_"BIC"__attr_"teststat"',
+            'autolag_"BIC"__attr_"pvalue"',
+            'autolag_"BIC"__attr_"usedlag"',
+        ]
 
         res = augmented_dickey_fuller(x=x, param=param)
         res = pd.Series(dict(res))
         self.assertCountEqual(list(res.index), expected_index)
-        self.assertGreater(res['attr_"pvalue"'], 0.10)
-        self.assertEqual(res['attr_"usedlag"'], 0)
+        self.assertGreater(res['autolag_"BIC"__attr_"pvalue"'], 0.10)
+        self.assertEqual(res['autolag_"BIC"__attr_"usedlag"'], 0)
 
         # H0 should be rejected for AR(1) model with x_{t} = 1/2 x_{t-1} + e_{t}
         np.random.seed(seed=42)
@@ -268,14 +276,22 @@ class FeatureCalculationTestCase(TestCase):
         x[0] = 100
         for i in range(1, m):
             x[i] = x[i - 1] * 0.5 + e[i]
-        param = [{"attr": "teststat"}, {"attr": "pvalue"}, {"attr": "usedlag"}]
-        expected_index = ['attr_"teststat"', 'attr_"pvalue"', 'attr_"usedlag"']
+        param = [
+            {"autolag": "AIC", "attr": "teststat"},
+            {"autolag": "AIC", "attr": "pvalue"},
+            {"autolag": "AIC", "attr": "usedlag"}
+        ]
+        expected_index = [
+            'autolag_"AIC"__attr_"teststat"',
+            'autolag_"AIC"__attr_"pvalue"',
+            'autolag_"AIC"__attr_"usedlag"',
+        ]
 
         res = augmented_dickey_fuller(x=x, param=param)
         res = pd.Series(dict(res))
         self.assertCountEqual(list(res.index), expected_index)
-        self.assertLessEqual(res['attr_"pvalue"'], 0.05)
-        self.assertEqual(res['attr_"usedlag"'], 0)
+        self.assertLessEqual(res['autolag_"AIC"__attr_"pvalue"'], 0.05)
+        self.assertEqual(res['autolag_"AIC"__attr_"usedlag"'], 0)
 
         # Check if LinAlgError and ValueError are catched
         res_linalg_error = augmented_dickey_fuller(x=np.repeat(np.nan, 100), param=param)
@@ -285,6 +301,11 @@ class FeatureCalculationTestCase(TestCase):
         for index, val in res_value_error:
             self.assertIsNaN(val)
 
+        # Should return NaN if "attr" is unknown
+        res_attr_error = augmented_dickey_fuller(x=x, param=[{"autolag": "AIC", "attr": ""}])
+        for index, val in res_attr_error:
+            self.assertIsNaN(val)
+
     def test_abs_energy(self):
         self.assertEqualOnAllArrayTypes(abs_energy, [1, 1, 1], 3)
         self.assertEqualOnAllArrayTypes(abs_energy, [1, 2, 3], 14)
diff --git a/tests/units/feature_selection/test_relevance.py b/tests/units/feature_selection/test_relevance.py
index bc47b90..a7a8b71 100644
--- a/tests/units/feature_selection/test_relevance.py
+++ b/tests/units/feature_selection/test_relevance.py
@@ -93,11 +93,13 @@ class TestCalculateRelevanceTable:
         significance_test_feature_binary_mock.return_value = 0.95
         significance_test_feature_real_mock.return_value = 0.95
 
-        with mock.patch('logging.Logger.warning') as m:
-            _ = calculate_relevance_table(X, y_real, n_jobs=0, ml_task="regression")
-            m.assert_called_with("No feature was found relevant for regression for fdr level = 0.05 (which corresponds "
-                                 "to the maximal percentage of irrelevant features, consider using an higher fdr level "
-                                 "or add other features.")
+        with pytest.warns(RuntimeWarning) as record:
+            _ = calculate_relevance_table(X, y_real, n_jobs=0, ml_task="regression", show_warnings=True)
+            assert len(record) == 1
+            assert str(record[0].message) == (
+                "No feature was found relevant for regression for fdr level = 0.05 (which corresponds "
+                "to the maximal percentage of irrelevant features, consider using an higher fdr level "
+                "or add other features.")
 
 
 class TestCombineRelevanceTables:
diff --git a/tsfresh/feature_selection/significance_tests.py b/tsfresh/feature_selection/significance_tests.py
index ccf6ec2..fe909a5 100644
--- a/tsfresh/feature_selection/significance_tests.py
+++ b/tsfresh/feature_selection/significance_tests.py
@@ -36,10 +36,7 @@ from builtins import str
 import numpy as np
 import pandas as pd
 from scipy import stats
-import logging
-
-
-_logger = logging.getLogger(__name__)
+import warnings
 
 
 def target_binary_feature_binary_test(x, y):
@@ -222,9 +219,9 @@ def __check_for_binary_target(y):
     :raises: ``ValueError`` if the values are not binary.
     """
     if not set(y) == {0, 1}:
-        _logger.warning("[target_binary_feature_binary_test] The binary target should have "
-                        "values 1 and 0 (or True and False). "
-                        "Instead found" + str(set(y)))
+        warnings.warn("[target_binary_feature_binary_test] The binary target should have "
+                      "values 1 and 0 (or True and False). "
+                      "Instead found" + str(set(y)), RuntimeWarning)
         if len(set(y)) > 2:
             raise ValueError("[target_binary_feature_binary_test] Target is not binary!")
 
@@ -243,9 +240,9 @@ def __check_for_binary_feature(x):
     :raises: ``ValueError`` if the values are not binary.
     """
     if not set(x) == {0, 1}:
-        _logger.warning("[target_binary_feature_binary_test] A binary feature should have only "
-                        "values 1 and 0 (incl. True and False). "
-                        "Instead found " + str(set(x)) + " in feature ''" + str(x.name) + "''.")
+        warnings.warn("[target_binary_feature_binary_test] A binary feature should have only "
+                      "values 1 and 0 (incl. True and False). "
+                      "Instead found " + str(set(x)) + " in feature ''" + str(x.name) + "''.", RuntimeWarning)
         if len(set(x)) > 2:
             raise ValueError("[target_binary_feature_binary_test] Feature is not binary!")