Repository: deepchecks/deepchecks

Issue:
[FEAT] add tests for more model types
we should test any model with predict and predict_proba functions, including the common ones:

Scikitlearn (also pipelines)
CatBoost
LGBM
XGBoost
Custom binary classification model that implements the predict_proba and predict functions
Custom multiclass classification model that implements the predict_proba and predict functions
Custom regression model that implements the predict function



Discussion:
need to think how we do it so we won't make dev installation huge, also many added tests for boosting overfit


Resolving patch:
diff --git a/deepchecks/tabular/checks/overview/model_info.py b/deepchecks/tabular/checks/overview/model_info.py
--- a/deepchecks/tabular/checks/overview/model_info.py
+++ b/deepchecks/tabular/checks/overview/model_info.py
@@ -35,8 +35,12 @@ def run_logic(self, context: Context) -> CheckResult:
         model = context.model
         estimator = get_model_of_pipeline(model)
         model_type = type(estimator).__name__
-        model_params = estimator.get_params()
-        default_params = type(estimator)().get_params()
+        try:
+            model_params = estimator.get_params()
+            default_params = type(estimator)().get_params()
+        except AttributeError:
+            model_params = {}
+            default_params = {}
 
         # Create dataframe to show
         model_param_df = pd.DataFrame(model_params.items(), columns=['Parameter', 'Value'])
diff --git a/deepchecks/tabular/checks/performance/confusion_matrix_report.py b/deepchecks/tabular/checks/performance/confusion_matrix_report.py
--- a/deepchecks/tabular/checks/performance/confusion_matrix_report.py
+++ b/deepchecks/tabular/checks/performance/confusion_matrix_report.py
@@ -9,6 +9,7 @@
 # ----------------------------------------------------------------------------
 #
 """The confusion_matrix_report check module."""
+import numpy as np
 import pandas as pd
 import sklearn
 import plotly.express as px
@@ -46,7 +47,7 @@ def run_logic(self, context: Context, dataset_type: str = 'train') -> CheckResul
         ds_x = dataset.features_columns
         model = context.model
 
-        y_pred = model.predict(ds_x)
+        y_pred = np.array(model.predict(ds_x)).reshape(len(ds_y), )
         total_classes = sorted(list(set(pd.concat([ds_y, pd.Series(y_pred)]).to_list())))
         confusion_matrix = sklearn.metrics.confusion_matrix(ds_y, y_pred)
 


Test patch:
diff --git a/tests/checks/methodology/boosting_overfit_test.py b/tests/checks/methodology/boosting_overfit_test.py
--- a/tests/checks/methodology/boosting_overfit_test.py
+++ b/tests/checks/methodology/boosting_overfit_test.py
@@ -40,8 +40,56 @@ def test_boosting_classifier(iris):
     test_scores = result.value['test']
     assert_that(train_scores, has_length(20))
     assert_that(test_scores, has_length(20))
-    assert_that(mean(train_scores), close_to(0.999, 0.01))
-    assert_that(mean(test_scores), close_to(0.961, 0.01))
+    assert_that(mean(train_scores), close_to(0.999, 0.001))
+    assert_that(mean(test_scores), close_to(0.961, 0.001))
+
+
+def test_boosting_xgb_classifier(iris_split_dataset_and_model_xgb):
+    # Arrange
+    train, test, clf = iris_split_dataset_and_model_xgb
+
+    # Act
+    result = BoostingOverfit().run(train, test, clf)
+
+    # Assert
+    train_scores = result.value['train']
+    test_scores = result.value['test']
+    assert_that(train_scores, has_length(20))
+    assert_that(test_scores, has_length(20))
+    assert_that(mean(train_scores), close_to(0.99, 0.001))
+    assert_that(mean(test_scores), close_to(0.985, 0.001))
+
+
+def test_boosting_lgbm_classifier(iris_split_dataset_and_model_lgbm):
+    # Arrange
+    train, test, clf = iris_split_dataset_and_model_lgbm
+
+    # Act
+    result = BoostingOverfit().run(train, test, clf)
+
+    # Assert
+    train_scores = result.value['train']
+    test_scores = result.value['test']
+    assert_that(train_scores, has_length(20))
+    assert_that(test_scores, has_length(20))
+    assert_that(mean(train_scores), close_to(0.972, 0.001))
+    assert_that(mean(test_scores), close_to(0.974, 0.001))
+
+
+def test_boosting_cat_classifier(iris_split_dataset_and_model_cat):
+    # Arrange
+    train, test, clf = iris_split_dataset_and_model_cat
+
+    # Act
+    result = BoostingOverfit().run(train, test, clf)
+
+    # Assert
+    train_scores = result.value['train']
+    test_scores = result.value['test']
+    assert_that(train_scores, has_length(20))
+    assert_that(test_scores, has_length(20))
+    assert_that(mean(train_scores), close_to(0.991, 0.001))
+    assert_that(mean(test_scores), close_to(0.979, 0.001))
 
 
 def test_boosting_model_is_pipeline(iris):
@@ -62,16 +110,16 @@ def test_boosting_model_is_pipeline(iris):
 
     assert_that(train_scores, has_length(20))
     assert_that(test_scores, has_length(20))
-    assert_that(mean(train_scores), close_to(0.999, 0.01))
-    assert_that(mean(test_scores), close_to(0.976, 0.01))
+    assert_that(mean(train_scores), close_to(0.999, 0.001))
+    assert_that(mean(test_scores), close_to(0.976, 0.001))
 
 
 def test_boosting_regressor(diabetes, diabetes_model):
     # Arrange
-    train, validation = diabetes
+    train, test = diabetes
 
     # Act
-    result = BoostingOverfit().run(train, validation, diabetes_model)
+    result = BoostingOverfit().run(train, test, diabetes_model)
 
     # Assert
     train_scores = result.value['train']
@@ -81,6 +129,54 @@ def test_boosting_regressor(diabetes, diabetes_model):
     assert_that(mean(train_scores), close_to(-44.52, 0.01))
     assert_that(mean(test_scores), close_to(-59.35, 0.01))
 
+def test_boosting_regressor_xgb(diabetes_split_dataset_and_model_xgb):
+    # Arrange
+    train, test, model = diabetes_split_dataset_and_model_xgb
+
+    # Act
+    result = BoostingOverfit().run(train, test, model)
+
+    # Assert
+    train_scores = result.value['train']
+    test_scores = result.value['test']
+    assert_that(train_scores, has_length(20))
+    assert_that(test_scores, has_length(20))
+    assert_that(mean(train_scores), close_to(-22.67, 0.01))
+    assert_that(mean(test_scores), close_to(-66.99, 0.01))
+
+
+def test_boosting_regressor_lgbm(diabetes_split_dataset_and_model_lgbm):
+    # Arrange
+    train, test, model = diabetes_split_dataset_and_model_lgbm
+
+    # Act
+    result = BoostingOverfit().run(train, test, model)
+
+    # Assert
+    train_scores = result.value['train']
+    test_scores = result.value['test']
+    assert_that(train_scores, has_length(20))
+    assert_that(test_scores, has_length(20))
+    assert_that(mean(train_scores), close_to(-41.46, 0.01))
+    assert_that(mean(test_scores), close_to(-59.87, 0.01))
+
+
+def test_boosting_regressor_cat(diabetes_split_dataset_and_model_cat):
+    # Arrange
+    train, test, model = diabetes_split_dataset_and_model_cat
+
+    # Act
+    result = BoostingOverfit().run(train, test, model)
+
+    # Assert
+    train_scores = result.value['train']
+    test_scores = result.value['test']
+    assert_that(train_scores, has_length(20))
+    assert_that(test_scores, has_length(20))
+    assert_that(mean(train_scores), close_to(-35.49, 0.01))
+    assert_that(mean(test_scores), close_to(-59.04, 0.01))
+
+
 
 def test_boosting_classifier_with_metric(iris):
     # Arrange
@@ -99,8 +195,8 @@ def test_boosting_classifier_with_metric(iris):
     test_scores = result.value['test']
     assert_that(train_scores, has_length(20))
     assert_that(test_scores, has_length(20))
-    assert_that(mean(train_scores), close_to(0.999, 0.01))
-    assert_that(mean(test_scores), close_to(0.96, 0.01))
+    assert_that(mean(train_scores), close_to(0.999, 0.001))
+    assert_that(mean(test_scores), close_to(0.961, 0.001))
 
 
 def test_condition_score_decline_not_greater_than_pass(diabetes, diabetes_model):
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,7 +11,7 @@
 """Represents fixtures for unit testing using pytest."""
 # Disable this pylint check since we use this convention in pytest fixtures
 #pylint: disable=redefined-outer-name
-from typing import Tuple
+from typing import Any, Tuple
 
 import numpy as np
 import pytest
@@ -24,6 +24,9 @@
 from sklearn.pipeline import Pipeline
 from sklearn.preprocessing import KBinsDiscretizer, OrdinalEncoder, FunctionTransformer
 from sklearn.tree import DecisionTreeClassifier
+from xgboost import XGBClassifier, XGBRegressor
+from catboost import CatBoostClassifier, CatBoostRegressor
+from lightgbm import LGBMClassifier, LGBMRegressor
 
 from deepchecks.tabular import Dataset, TrainTestCheck, Context
 from deepchecks.core.check_result import CheckResult
@@ -109,6 +112,42 @@ def diabetes_split_dataset_and_model(diabetes, diabetes_model):
     return train, test, clf
 
 
+@pytest.fixture(scope='session')
+def diabetes_split_dataset_and_model_custom(diabetes, diabetes_model):
+    train, test = diabetes
+    class MyModel:
+        def predict(self, *args, **kwargs):
+            return diabetes_model.predict(*args, **kwargs)
+        # sklearn scorers in python 3.6 check fit attr
+        def fit(self, *args, **kwargs):
+            return diabetes_model.fit(*args, **kwargs)
+    return train, test, MyModel()
+
+
+@pytest.fixture(scope='session')
+def diabetes_split_dataset_and_model_xgb(diabetes):
+    train, test = diabetes
+    clf = XGBRegressor(random_state=0)
+    clf.fit(train.data[train.features], train.data[train.label_name])
+    return train, test, clf
+
+
+@pytest.fixture(scope='session')
+def diabetes_split_dataset_and_model_lgbm(diabetes):
+    train, test = diabetes
+    clf = LGBMRegressor(random_state=0)
+    clf.fit(train.data[train.features], train.data[train.label_name])
+    return train, test, clf
+
+
+@pytest.fixture(scope='session')
+def diabetes_split_dataset_and_model_cat(diabetes):
+    train, test = diabetes
+    clf = CatBoostRegressor(random_state=0)
+    clf.fit(train.data[train.features], train.data[train.label_name], verbose=False)
+    return train, test, clf
+
+
 @pytest.fixture(scope='session')
 def iris_clean():
     """Return Iris dataset as DataFrame."""
@@ -190,17 +229,40 @@ def iris_dataset_single_class_labeled(iris):
     return dataset
 
 
-@pytest.fixture
-def iris_split_dataset_and_model(iris_clean) -> Tuple[Dataset, Dataset, AdaBoostClassifier]:
+@pytest.fixture(scope='session')
+def iris_split_dataset(iris_clean) -> Tuple[Dataset, Dataset]:
     """Return Iris train and val datasets and trained AdaBoostClassifier model."""
     train, test = train_test_split(iris_clean.frame, test_size=0.33, random_state=42)
     train_ds = Dataset(train, label='target')
     test_ds = Dataset(test, label='target')
+    return train_ds, test_ds
+
+
+@pytest.fixture(scope='session')
+def iris_split_dataset_and_model(iris_split_dataset) -> Tuple[Dataset, Dataset, AdaBoostClassifier]:
+    """Return Iris train and val datasets and trained AdaBoostClassifier model."""
+    train_ds, test_ds = iris_split_dataset
     clf = AdaBoostClassifier(random_state=0)
     clf.fit(train_ds.features_columns, train_ds.label_col)
     return train_ds, test_ds, clf
 
-@pytest.fixture
+
+@pytest.fixture(scope='session')
+def iris_split_dataset_and_model_custom(iris_split_dataset_and_model) -> Tuple[Dataset, Dataset, Any]:
+    """Return Iris train and val datasets and trained AdaBoostClassifier model."""
+    train_ds, test_ds, clf = iris_split_dataset_and_model
+    class MyModel:
+        def predict(self, *args, **kwargs):
+            return clf.predict(*args, **kwargs)
+        def predict_proba(self, *args, **kwargs):
+            return clf.predict_proba(*args, **kwargs)
+        # sklearn scorers in python 3.6 check fit attr
+        def fit(self, *args, **kwargs):
+            return clf.fit(*args, **kwargs)
+    return train_ds, test_ds, MyModel()
+
+
+@pytest.fixture(scope='session')
 def iris_split_dataset_and_model_single_feature(iris_clean) -> Tuple[Dataset, Dataset, AdaBoostClassifier]:
     """Return Iris train and val datasets and trained AdaBoostClassifier model."""
     train, test = train_test_split(iris_clean.frame, test_size=0.33, random_state=42)
@@ -213,17 +275,42 @@ def iris_split_dataset_and_model_single_feature(iris_clean) -> Tuple[Dataset, Da
 
 
 @pytest.fixture(scope='session')
-def iris_split_dataset_and_model_rf(iris) -> Tuple[Dataset, Dataset, RandomForestClassifier]:
+def iris_split_dataset_and_model_xgb(iris_split_dataset) -> Tuple[Dataset, Dataset, XGBClassifier]:
+    """Return Iris train and val datasets and trained AdaBoostClassifier model."""
+    train_ds, test_ds = iris_split_dataset
+    clf = XGBClassifier(random_state=0)
+    clf.fit(train_ds.features_columns, train_ds.label_col)
+    return train_ds, test_ds, clf
+
+
+@pytest.fixture(scope='session')
+def iris_split_dataset_and_model_lgbm(iris_split_dataset) -> Tuple[Dataset, Dataset, LGBMClassifier]:
+    """Return Iris train and val datasets and trained AdaBoostClassifier model."""
+    train_ds, test_ds = iris_split_dataset
+    clf = LGBMClassifier(random_state=0)
+    clf.fit(train_ds.features_columns, train_ds.label_col)
+    return train_ds, test_ds, clf
+
+
+@pytest.fixture(scope='session')
+def iris_split_dataset_and_model_cat(iris_split_dataset) -> Tuple[Dataset, Dataset, CatBoostClassifier]:
+    """Return Iris train and val datasets and trained AdaBoostClassifier model."""
+    train_ds, test_ds = iris_split_dataset
+    clf = CatBoostClassifier(random_state=0)
+    clf.fit(train_ds.features_columns, train_ds.label_col, verbose=False)
+    return train_ds, test_ds, clf
+
+
+@pytest.fixture(scope='session')
+def iris_split_dataset_and_model_rf(iris_split_dataset) -> Tuple[Dataset, Dataset, RandomForestClassifier]:
     """Return Iris train and val datasets and trained RF model."""
-    train, test = train_test_split(iris, test_size=0.33, random_state=0)
-    train_ds = Dataset(train, label='target')
-    test_ds = Dataset(test, label='target')
+    train_ds, test_ds = iris_split_dataset
     clf = RandomForestClassifier(random_state=0, n_estimators=10, max_depth=2)
     clf.fit(train_ds.features_columns, train_ds.label_col)
     return train_ds, test_ds, clf
 
 
-@pytest.fixture
+@pytest.fixture(scope='session')
 def iris_binary_string_split_dataset_and_model(iris) -> Tuple[Dataset, Dataset, DecisionTreeClassifier]:
     """Return Iris train and test datasets and trained DecisionTreeClassifier model."""
     iris = iris.copy()
diff --git a/tests/suites/test_suites.py b/tests/suites/test_suites.py
--- a/tests/suites/test_suites.py
+++ b/tests/suites/test_suites.py
@@ -11,6 +11,8 @@
 """builtin suites tests"""
 #pylint: disable=redefined-outer-name
 import typing as t
+from catboost import CatBoostClassifier, CatBoostRegressor
+from lightgbm import LGBMClassifier, LGBMRegressor
 import pytest
 from datetime import datetime
 
@@ -19,6 +21,7 @@
 from sklearn.model_selection import train_test_split
 from hamcrest.core.matcher import Matcher
 from hamcrest import assert_that, instance_of, only_contains, any_of
+from xgboost import XGBClassifier, XGBRegressor
 
 from deepchecks.core import SuiteResult, CheckResult, CheckFailure
 from deepchecks.tabular import suites, Dataset, SingleDatasetCheck, Suite
@@ -79,6 +82,59 @@ def test_generic_suite(
         validate_suite_result(result, length)
 
 
+def test_generic_boost(
+    iris_split_dataset_and_model_cat: t.Tuple[Dataset, Dataset, CatBoostClassifier],
+    iris_split_dataset_and_model_xgb: t.Tuple[Dataset, Dataset, XGBClassifier],
+    iris_split_dataset_and_model_lgbm : t.Tuple[Dataset, Dataset, LGBMClassifier],
+    diabetes_split_dataset_and_model_xgb: t.Tuple[Dataset, Dataset, CatBoostRegressor],
+    diabetes_split_dataset_and_model_lgbm: t.Tuple[Dataset, Dataset, XGBRegressor],
+    diabetes_split_dataset_and_model_cat : t.Tuple[Dataset, Dataset, LGBMRegressor],
+):
+    iris_cat_train, iris_cat_test, iris_cat_model = iris_split_dataset_and_model_cat
+    iris_xgb_train, iris_xgb_test, iris_xgb_model = iris_split_dataset_and_model_xgb
+    iris_lgbm_train, iris_lgbm_test, iris_lgbm_model = iris_split_dataset_and_model_lgbm
+
+    diabetes_cat_train, diabetes_cat_test, diabetes_cat_model = diabetes_split_dataset_and_model_cat
+    diabetes_xgb_train, diabetes_xgb_test, diabetes_xgb_model = diabetes_split_dataset_and_model_xgb
+    diabetes_lgbm_train, diabetes_lgbm_test, diabetes_lgbm_model = diabetes_split_dataset_and_model_lgbm
+
+    suite = suites.full_suite()
+
+    arguments = (
+        dict(train_dataset=iris_cat_train, test_dataset=iris_cat_test, model=iris_cat_model),
+        dict(train_dataset=iris_xgb_train, test_dataset=iris_xgb_test, model=iris_xgb_model),
+        dict(train_dataset=iris_lgbm_train, test_dataset=iris_lgbm_test, model=iris_lgbm_model),
+        dict(train_dataset=diabetes_cat_train, test_dataset=diabetes_cat_test, model=diabetes_cat_model),
+        dict(train_dataset=diabetes_xgb_train, test_dataset=diabetes_xgb_test, model=diabetes_xgb_model),
+        dict(train_dataset=diabetes_lgbm_train, test_dataset=diabetes_lgbm_test, model=diabetes_lgbm_model),
+    )
+
+    for args in arguments:
+        result = suite.run(**args)
+        length = get_expected_results_length(suite, args)
+        validate_suite_result(result, length)
+
+
+def test_generic_custom(
+    iris_split_dataset_and_model_custom: t.Tuple[Dataset, Dataset, t.Any],
+    diabetes_split_dataset_and_model_custom: t.Tuple[Dataset, Dataset, t.Any],
+):
+    iris_train, iris_test, iris_model = iris_split_dataset_and_model_custom
+    diabetes_train, diabetes_test, diabetes_model = diabetes_split_dataset_and_model_custom
+
+    suite = suites.full_suite()
+
+    arguments = (
+        dict(train_dataset=iris_train, test_dataset=iris_test, model=iris_model),
+        dict(train_dataset=diabetes_train, test_dataset=diabetes_test, model=diabetes_model),
+    )
+
+    for args in arguments:
+        result = suite.run(**args)
+        length = get_expected_results_length(suite, args)
+        validate_suite_result(result, length)
+
+
 def validate_suite_result(
     result: SuiteResult,
     min_length: int,