Repository: dabl/dabl

Issue:
URGENT: ImportError Issue with `detect_types` Module from `dabl` Library
Dear experts and fellow users,

I'm currently facing an issue with the `detect_types` module from the `dabl` library. Despite having ensured that both `dabl` and `sklearn` are up to date, I continue to encounter an ImportError.

Upon investigating, I discovered that the issue stems from the deprecation of the function 'if_delegate_has_method' in `sklearn.utils.metaestimators`, scheduled for removal in version 1.3.0 of scikit-learn.

I'm now seeking guidance on how to resolve this dilemma and successfully run the code. Any assistance or insights would be greatly appreciated!

Thank you in advance for your valuable assistance.
![WX20230902-194117@2x](https://github.com/dabl/dabl/assets/63391137/cb315b4c-3c2d-429e-89d8-efcc123e744b)


Comments:

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index eed6444..1b30b76 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -43,7 +43,8 @@ jobs:
       - name: Show versions and install dabl
         run: |
           python --version
-          python setup.py develop
+          #python setup.py develop  # deprecated syntax
+          python -m pip install -e .
           python -c "import numpy; print('numpy %s' % numpy.__version__)"
           python -c "import scipy; print('scipy %s' % scipy.__version__)"
           python -c "import matplotlib; print('matplotlib %s' % matplotlib.__version__)"
diff --git a/.gitignore b/.gitignore
index 0bd5ca1..616d604 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@ __pycache__/
 *.py[cod]
 *$py.class
 .vscode
+.venv
 coverage
 
 # C extensions
diff --git a/dabl/_available_if.py b/dabl/_available_if.py
new file mode 100644
index 0000000..ce564ba
--- /dev/null
+++ b/dabl/_available_if.py
@@ -0,0 +1,90 @@
+# Note: As a part of dabl PR #335, this file was copied from
+# sklearn/utils/_available_if.py from the scikit-learn repo at commit
+# e2b3785b6a1f96989c3992bffa2a05ef5c048a7e
+
+from functools import update_wrapper, wraps
+from types import MethodType
+
+
+class _AvailableIfDescriptor:
+    """Implements a conditional property using the descriptor protocol.
+
+    Using this class to create a decorator will raise an ``AttributeError``
+    if check(self) returns a falsey value. Note that if check raises an error
+    this will also result in hasattr returning false.
+
+    See https://docs.python.org/3/howto/descriptor.html for an explanation of
+    descriptors.
+    """
+
+    def __init__(self, fn, check, attribute_name):
+        self.fn = fn
+        self.check = check
+        self.attribute_name = attribute_name
+
+        # update the docstring of the descriptor
+        update_wrapper(self, fn)
+
+    def __get__(self, obj, owner=None):
+        attr_err = AttributeError(
+            f"This {repr(owner.__name__)} has no attribute {repr(self.attribute_name)}"
+        )
+        if obj is not None:
+            # delegate only on instances, not the classes.
+            # this is to allow access to the docstrings.
+            if not self.check(obj):
+                raise attr_err
+            out = MethodType(self.fn, obj)
+
+        else:
+            # This makes it possible to use the decorated method as an unbound method,
+            # for instance when monkeypatching.
+            @wraps(self.fn)
+            def out(*args, **kwargs):
+                if not self.check(args[0]):
+                    raise attr_err
+                return self.fn(*args, **kwargs)
+
+        return out
+
+
+def available_if(check):
+    """An attribute that is available only if check returns a truthy value.
+
+    Parameters
+    ----------
+    check : callable
+        When passed the object with the decorated method, this should return
+        a truthy value if the attribute is available, and either return False
+        or raise an AttributeError if not available.
+
+    Returns
+    -------
+    callable
+        Callable makes the decorated method available if `check` returns
+        a truthy value, otherwise the decorated method is unavailable.
+
+    Examples
+    --------
+    >>> from sklearn.utils.metaestimators import available_if
+    >>> class HelloIfEven:
+    ...    def __init__(self, x):
+    ...        self.x = x
+    ...
+    ...    def _x_is_even(self):
+    ...        return self.x % 2 == 0
+    ...
+    ...    @available_if(_x_is_even)
+    ...    def say_hello(self):
+    ...        print("Hello")
+    ...
+    >>> obj = HelloIfEven(1)
+    >>> hasattr(obj, "say_hello")
+    False
+    >>> obj.x = 2
+    >>> hasattr(obj, "say_hello")
+    True
+    >>> obj.say_hello()
+    Hello
+    """
+    return lambda fn: _AvailableIfDescriptor(fn, check, attribute_name=fn.__name__)
diff --git a/dabl/models.py b/dabl/models.py
index 7dd1c39..e3a42f7 100644
--- a/dabl/models.py
+++ b/dabl/models.py
@@ -1,3 +1,4 @@
+from importlib.metadata import version
 import warnings
 import numpy as np
 import pandas as pd
@@ -17,7 +18,7 @@ except ImportError:
     from sklearn.metrics.scorer import _check_multimetric_scoring
 from sklearn.model_selection._validation import _fit_and_score
 from sklearn.utils.validation import check_is_fitted
-from sklearn.utils.metaestimators import if_delegate_has_method
+from dabl._available_if import available_if
 try:
     from sklearn.utils._testing import set_random_state
 except ImportError:
@@ -33,6 +34,8 @@ from .pipelines import (get_fast_classifiers, get_fast_regressors,
                         get_any_classifiers)
 from .utils import _validate_Xyt
 
+_SKLEARN_VERSION = version('scikit-learn')
+
 
 def _format_scores(scores):
     return " ".join(('{}: {:.3f}'.format(name, score)
@@ -41,11 +44,14 @@ def _format_scores(scores):
 
 class _DablBaseEstimator(BaseEstimator):
 
-    @if_delegate_has_method(delegate='est_')
+    def _estimator_has(attr: str) -> bool:
+        return lambda self: hasattr(self, 'est_') and hasattr(self.est_, attr)
+
+    @available_if(_estimator_has('predict_proba'))
     def predict_proba(self, X):
         return self.est_.predict_proba(X)
 
-    @if_delegate_has_method(delegate='est_')
+    @available_if(_estimator_has('decision_function'))
     def decision_function(self, X):
         return self.est_.decision_function(X)
 
@@ -76,10 +82,11 @@ class _BaseSimpleEstimator(_DablBaseEstimator):
             with warnings.catch_warnings():
                 warnings.filterwarnings('ignore',
                                         category=UndefinedMetricWarning)
+                _fit_and_score_args = {'score_params': {}} if _SKLEARN_VERSION >= '1.4' else {}
                 scores = _fit_and_score(estimator, X, y, scorer=scorers,
                                         train=train, test=test,
                                         parameters={}, fit_params={},
-                                        verbose=self.verbose)
+                                        verbose=self.verbose, **_fit_and_score_args)
             res.append(scores['test_scores'])
 
         res_mean = pd.DataFrame(res).mean(axis=0)
diff --git a/dabl/plot/utils.py b/dabl/plot/utils.py
index b49f2b6..07b264b 100644
--- a/dabl/plot/utils.py
+++ b/dabl/plot/utils.py
@@ -6,6 +6,7 @@ import itertools
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
+from matplotlib.axes import Axes
 from matplotlib.patches import Rectangle, Patch
 from matplotlib.ticker import EngFormatter
 from seaborn.utils import despine
@@ -369,7 +370,7 @@ def _check_X_target_col(X, target_col, types=None, type_hints=None, task=None):
     return types
 
 
-def _short_tick_names(ax, label_length=20, ticklabel_length=10):
+def _short_tick_names(ax: Axes, label_length=20, ticklabel_length=10):
     """Shorten axes labels and tick labels.
 
     Uses _shortname to change labels as a side effect.
@@ -384,8 +385,19 @@ def _short_tick_names(ax, label_length=20, ticklabel_length=10):
         Length of each label in xticklabels and yticklabels
 
     """
-    ax.set_yticks(ax.get_yticks().tolist())
-    ax.set_xticks(ax.get_xticks().tolist())
+    # Handle differences in return types between matplotlib 3.7 and 3.8
+    yticks = ax.get_yticks()
+    if isinstance(yticks, np.ndarray):
+        yticks = yticks.tolist()
+    assert isinstance(yticks, list)
+    ax.set_yticks(yticks)
+
+    xticks = ax.get_xticks()
+    if isinstance(xticks, np.ndarray):
+        xticks = xticks.tolist()
+    assert isinstance(xticks, list)
+    ax.set_xticks(xticks)
+
     ax.set_xticklabels(
         [_shortname(t.get_text(), maxlen=ticklabel_length)
          for t in ax.get_xticklabels()]
@@ -578,7 +590,7 @@ def class_hists(data, column, target, bins="auto", ax=None, legend=True,
     >>> from dabl.datasets import load_adult
     >>> data = load_adult()
     >>> class_hists(data, "age", "gender", legend=True)
-    <AxesSubplot:xlabel='age'>
+    <Axes: xlabel='age'>
     """
     col_data = data[column].dropna()
 
diff --git a/dabl/preprocessing.py b/dabl/preprocessing.py
index 67c43d8..055d67a 100644
--- a/dabl/preprocessing.py
+++ b/dabl/preprocessing.py
@@ -1,3 +1,4 @@
+from importlib.metadata import version
 from joblib import hash
 import warnings
 from warnings import warn
@@ -11,10 +12,13 @@ from sklearn.pipeline import make_pipeline
 from sklearn.preprocessing import OneHotEncoder, StandardScaler
 from sklearn.utils.validation import check_is_fitted
 
+
 _FLOAT_REGEX = r"^[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))$"
 _FLOAT_MATCHING_CACHE = {}
 _MIXED_TYPE_WARNINGS = {}
 
+_SKLEARN_VERSION = version('scikit-learn')
+
 
 def _float_matching(X_col, return_safe_col=False):
     try:
@@ -86,7 +90,8 @@ class DirtyFloatCleaner(BaseEstimator, TransformerMixin):
             floats, X_col = _float_matching_fetch(X, col, return_safe_col=True)
             # FIXME sparse
             if (~floats).any():
-                ohe = OneHotEncoder(sparse=False, handle_unknown='ignore')
+                ohe_args = {'sparse_output': False} if _SKLEARN_VERSION >= '1.2' else {'sparse': False}
+                ohe = OneHotEncoder(handle_unknown='ignore', **ohe_args)
                 encoders[col] = ohe.fit(pd.DataFrame(X_col[~floats]))
             else:
                 encoders[col] = None
@@ -601,9 +606,10 @@ class EasyPreprocessor(BaseEstimator, TransformerMixin):
             steps_categorical.append(
                 SimpleImputer(strategy='most_frequent', add_indicator=True)
                 )
+            ohe_args = {'sparse_output': False} if _SKLEARN_VERSION >= '1.2' else {'sparse': False}
             ohe = OneHotEncoder(
                 categories='auto', handle_unknown='ignore',
-                sparse=False, drop="if_binary")
+                drop="if_binary", **ohe_args)
             steps_categorical.append(ohe)
 
         pipe_categorical = make_pipeline(*steps_categorical)
diff --git a/setup.py b/setup.py
index 6eca18d..b0141ec 100644
--- a/setup.py
+++ b/setup.py
@@ -13,8 +13,16 @@ setup(name='dabl',
       long_description=readme,
       long_description_content_type='text/markdown; charset=UTF-8',
       packages=find_packages(),
-      install_requires=["numpy", "scipy", "scikit-learn>=1.1", "pandas",
-                        "matplotlib>=3.5", "seaborn"],
+      install_requires=[
+          "numpy",
+          "scipy",
+          "scikit-learn>=1.1",
+          "pandas",
+          "matplotlib < 3.8;python_version<'3.9'",
+          "matplotlib >= 3.8;python_version>='3.9'",
+          "seaborn",
+      ],
+      python_requires=">=3.8",
       author_email='t3kcit+githubspam@gmail.com',
       package_data={'': ['datasets/titanic.csv',
                          'datasets/ames_housing.pkl.bz2',


Test patch:
diff --git a/dabl/plot/tests/test_supervised.py b/dabl/plot/tests/test_supervised.py
index 57b4fe3..4718687 100644
--- a/dabl/plot/tests/test_supervised.py
+++ b/dabl/plot/tests/test_supervised.py
@@ -27,27 +27,38 @@ from dabl import set_config
 def test_plots_smoke(continuous_features, categorical_features, task):
     # simple smoke test
     # should be parametrized
+    if continuous_features == 0 and categorical_features == 0:
+        pytest.skip("Need at least one feature")
     n_samples = 100
-    X_cont, y_cont = make_regression(
-        n_samples=n_samples, n_features=continuous_features,
-        n_informative=min(continuous_features, 2))
-    X_cat, y_cat = make_regression(
-        n_samples=n_samples, n_features=categorical_features,
-        n_informative=min(categorical_features, 2))
-    if X_cat.shape[1] > 0:
-        X_cat = KBinsDiscretizer(encode='ordinal').fit_transform(X_cat)
-    cont_columns = ["asdf_%d_cont" % i for i in range(continuous_features)]
-    df_cont = pd.DataFrame(X_cont, columns=cont_columns)
+    if continuous_features > 0:
+        X_cont, y_cont = make_regression(
+            n_samples=n_samples, n_features=continuous_features,
+            n_informative=min(continuous_features, 2))
+    if categorical_features > 0:
+        X_cat, y_cat = make_regression(
+            n_samples=n_samples, n_features=categorical_features,
+            n_informative=min(categorical_features, 2))
+    if continuous_features > 0:
+        cont_columns = ["asdf_%d_cont" % i for i in range(continuous_features)]
+        df_cont = pd.DataFrame(X_cont, columns=cont_columns)
     if categorical_features > 0:
+        X_cat = KBinsDiscretizer(encode='ordinal').fit_transform(X_cat)
         cat_columns = ["asdf_%d_cat" % i for i in range(categorical_features)]
         df_cat = pd.DataFrame(X_cat, columns=cat_columns).astype('int')
         df_cat = df_cat.astype("category")
+    if categorical_features > 0 and continuous_features > 0:
         X_df = pd.concat([df_cont, df_cat], axis=1)
-    else:
+        y = y_cont + y_cat
+    elif categorical_features > 0:
+        X_df = df_cat
+        y = y_cat
+    elif continuous_features > 0:
         X_df = df_cont
+        y = y_cont
+    else:
+        raise ValueError("invalid")
     assert X_df.shape[1] == continuous_features + categorical_features
     X_clean = clean(X_df.copy())
-    y = y_cont + y_cat
     if X_df.shape[1] == 0:
         y = np.random.uniform(size=n_samples)
     if task == "classification":
@@ -73,9 +84,9 @@ def test_plots_smoke(continuous_features, categorical_features, task):
                                            ['continuous', 'categorical'],
                                            ['continuous', 'categorical']))
 def test_type_hints(add, feature_type, target_type):
-    X = pd.DataFrame(np.random.randint(4, size=100)) + add
+    X = pd.DataFrame(np.random.randint(4, size=100), columns=['feat']) + add
     X['target'] = np.random.uniform(size=100)
-    plot(X, type_hints={0: feature_type,
+    plot(X, type_hints={'feat': feature_type,
                         'target': target_type},
          target_col='target')
     # get title of figure
diff --git a/dabl/tests/test_available_if.py b/dabl/tests/test_available_if.py
new file mode 100644
index 0000000..66fc221
--- /dev/null
+++ b/dabl/tests/test_available_if.py
@@ -0,0 +1,67 @@
+# Note: As a part of dabl PR #335, this file was copied from
+# sklearn/utils/tests/test_metaestimators.py from the scikit-learn repo at commit
+# e2b3785b6a1f96989c3992bffa2a05ef5c048a7e
+
+import pickle
+
+import pytest
+
+from dabl._available_if import available_if
+
+
+class AvailableParameterEstimator:
+    """This estimator's `available` parameter toggles the presence of a method"""
+
+    def __init__(self, available=True, return_value=1):
+        self.available = available
+        self.return_value = return_value
+
+    @available_if(lambda est: est.available)
+    def available_func(self):
+        """This is a mock available_if function"""
+        return self.return_value
+
+
+def test_available_if_docstring():
+    assert "This is a mock available_if function" in str(
+        AvailableParameterEstimator.__dict__["available_func"].__doc__
+    )
+    assert "This is a mock available_if function" in str(
+        AvailableParameterEstimator.available_func.__doc__
+    )
+    assert "This is a mock available_if function" in str(
+        AvailableParameterEstimator().available_func.__doc__
+    )
+
+
+def test_available_if():
+    assert hasattr(AvailableParameterEstimator(), "available_func")
+    assert not hasattr(AvailableParameterEstimator(available=False), "available_func")
+
+
+def test_available_if_unbound_method():
+    # This is a non regression test for:
+    # https://github.com/scikit-learn/scikit-learn/issues/20614
+    # to make sure that decorated functions can be used as an unbound method,
+    # for instance when monkeypatching.
+    est = AvailableParameterEstimator()
+    AvailableParameterEstimator.available_func(est)
+
+    est = AvailableParameterEstimator(available=False)
+    with pytest.raises(
+        AttributeError,
+        match="This 'AvailableParameterEstimator' has no attribute 'available_func'",
+    ):
+        AvailableParameterEstimator.available_func(est)
+
+
+def test_available_if_methods_can_be_pickled():
+    """Check that available_if methods can be pickled.
+
+    Non-regression test for #21344.
+    """
+    return_value = 10
+    est = AvailableParameterEstimator(available=True, return_value=return_value)
+    pickled_bytes = pickle.dumps(est.available_func)
+    unpickled_func = pickle.loads(pickled_bytes)
+    assert unpickled_func() == return_value