internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: deepchecks/deepchecks

Issue:
[Refactor] Identifier Leakage Check should be plotted with plotly and not matplotlib



Discussion:

Resolving patch:
diff --git a/deepchecks/checks/methodology/identifier_leakage.py b/deepchecks/checks/methodology/identifier_leakage.py
--- a/deepchecks/checks/methodology/identifier_leakage.py
+++ b/deepchecks/checks/methodology/identifier_leakage.py
@@ -16,10 +16,9 @@
 import deepchecks.ppscore as pps
 from deepchecks import Dataset
 from deepchecks.base.check import CheckResult, SingleDatasetBaseCheck, ConditionResult
-from deepchecks.utils.plot import create_colorbar_barchart_for_check
 from deepchecks.utils.strings import format_percent
 from deepchecks.errors import DeepchecksValueError
-
+import plotly.express as px
 
 __all__ = ['IdentifierLeakage']
 
@@ -67,18 +66,38 @@ def _identifier_leakage(self, dataset: Union[pd.DataFrame, Dataset], ppscore_par
         df_pps = df_pps.set_index('x', drop=True)
         s_ppscore = df_pps['ppscore']
 
-        def plot():
-            # Create graph:
-            create_colorbar_barchart_for_check(x=s_ppscore.index, y=s_ppscore.values,
-                                               ylabel='predictive power score (PPS)',
-                                               xlabel='Identifiers', color_map='gist_heat_r', color_shift_midpoint=0.1,
-                                               color_label='PPS', check_name=self.__class__.__name__)
+        xaxis_layout = dict(title='Identifiers', type='category')
+        yaxis_layout = dict(fixedrange=True,
+                            range=(0, 1),
+                            title='predictive power score (PPS)')
+
+        red_heavy_colorscale = [
+            [0, 'rgb(255, 255, 255)'],  # jan
+            [0.1, 'rgb(255,155,100)'],
+            [0.2, 'rgb(255, 50, 50)'],
+            [0.3, 'rgb(200, 0, 0)'],
+            [1, 'rgb(55, 0, 0)']
+        ]
+
+        figure = px.bar(s_ppscore, x=s_ppscore.index, y='ppscore', color='ppscore',
+                        color_continuous_scale=red_heavy_colorscale)
+        figure.update_layout(width=700, height=400)
+        figure.update_layout(
+            dict(
+                xaxis=xaxis_layout,
+                yaxis=yaxis_layout,
+                coloraxis=dict(
+                    cmin=0,
+                    cmax=1
+                )
+            )
+        )
 
         text = ['The PPS represents the ability of a feature to single-handedly predict another feature or label.',
                 'For Identifier columns (Index/Date) PPS should be nearly 0, otherwise date and index have some '
                 'predictive effect on the label.']
 
-        return CheckResult(value=s_ppscore.to_dict(), display=[plot, *text])
+        return CheckResult(value=s_ppscore.to_dict(), display=[figure, *text])
 
     def add_condition_pps_not_greater_than(self, max_pps: float = 0):
         """Add condition - require columns not to have a greater pps than given max.


Test patch:
Repository: deepchecks/deepchecks

Issue:
[BUG] Model Identification in sklearn Pipeline should look at the last step
```
pipe = Pipeline(steps=[
    ('transform', transformers),
    ('handle_nans', SimpleImputer(strategy='most_frequent')),
    ('modle', clf)
])

```

Boosting Overfit | DeepchecksValueError: Unsupported model of type: SimpleImputer
-- | --

Boosting Overfit check should have identified the pipeline as the last step (`clf` which was `catboost`)




Discussion:

Resolving patch:
diff --git a/deepchecks/utils/model.py b/deepchecks/utils/model.py
--- a/deepchecks/utils/model.py
+++ b/deepchecks/utils/model.py
@@ -14,8 +14,6 @@
 from sklearn.pipeline import Pipeline
 from sklearn.base import BaseEstimator
 
-from deepchecks.errors import DeepchecksValueError
-
 
 __all__ = ['get_model_of_pipeline']
 
@@ -29,9 +27,6 @@ def get_model_of_pipeline(model: Union[Pipeline, BaseEstimator]):
         the inner BaseEstimator of the Pipeline or itself
     """
     if isinstance(model, Pipeline):
-        # get feature importance from last model in pipeline
-        internal_estimator_list = [x[1] for x in model.steps if isinstance(x[1], BaseEstimator)]
-        if internal_estimator_list:
-            return internal_estimator_list[-1]
-        raise DeepchecksValueError('Received a pipeline without an sklearn compatible model')
+        # get model type from last step in pipeline
+        return model.steps[-1][1]
     return model
diff --git a/deepchecks/utils/validation.py b/deepchecks/utils/validation.py
--- a/deepchecks/utils/validation.py
+++ b/deepchecks/utils/validation.py
@@ -16,6 +16,7 @@
 
 from deepchecks import base  # pylint: disable=unused-import, is used in type annotations
 from deepchecks import errors
+from deepchecks.utils.model import get_model_of_pipeline
 from deepchecks.utils.typing import Hashable
 
 
@@ -30,6 +31,8 @@ def model_type_validation(model: t.Any):
     """
     supported_by_class_name = ('CatBoostClassifier', 'CatBoostRegressor')
     supported_by_class_instance = (sklearn.base.BaseEstimator,)
+    model = get_model_of_pipeline(model)
+
     if (
         not isinstance(model, supported_by_class_instance)
         and model.__class__.__name__ not in supported_by_class_name


Test patch:
Repository: deepchecks/deepchecks

Issue:
[BUG] Documentation - Examples - structure and check demonstration examples
**Bug Description**
Two-in-One:
1. Name of sections disappeared from sidebar ("How-to Guides, Check Demonstrations", and soon - "Use Cases").
Please note it should be "How-to Guides" and not "Guides".
See screenshot (pink)

2. Notebooks have no title:
- Model Error Analysis
- Multi Model Performance Report
See screenshot (highlight)

**To Reproduce**
see:
https://docs.deepchecks.com/en/latest/examples/index.html

**Screenshots**
![image](https://user-images.githubusercontent.com/33841818/148229836-c38f78f5-d86e-42e8-93b6-d0ff694ca333.png)




Discussion:

Resolving patch:
diff --git a/docs/source/conf.py b/docs/source/conf.py
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -50,9 +50,10 @@
 #
 nitpicky = True
 
-# Path to logo
+# Path to logo and favicon
 #
 html_logo = "./_static/deepchecks_logo.svg"
+html_favicon = "./_static/favicons/favicon.ico"
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom


Test patch:
Repository: deepchecks/deepchecks

Issue:
[BUG] inconsistent numbers of samples in SimpleModelComparison
**Describe the bug**
When running the SimpleModelComparison on the [url phishing dataset](https://github.com/deepchecks/deepchecks/blob/main/docs/source/examples/use-cases/phishing_urls.ipynb), the following error is raised:

`ValueError: Found input variables with inconsistent numbers of samples: [2724, 83682]`

**To Reproduce**
from deepchecks.checks import SimpleModelComparison
from deepchecks import Dataset
SimpleModelComparison().run(Dataset(train_df, label='target'), Dataset(test_df, label='target'), model)

**Screenshots**
![image](https://user-images.githubusercontent.com/17730502/148388818-c0df51f3-6a1b-4a4a-8557-4a2890aca409.png)
**Environment (please complete the following information):**
 - OS: wsl
 - 3.8.5
 - 0.2.0



Discussion:

Resolving patch:
diff --git a/deepchecks/checks/performance/simple_model_comparison.py b/deepchecks/checks/performance/simple_model_comparison.py
--- a/deepchecks/checks/performance/simple_model_comparison.py
+++ b/deepchecks/checks/performance/simple_model_comparison.py
@@ -144,7 +144,7 @@ def run(
         # Multiclass have different return type from the scorer, list of score per class instead of single score
         if task_type in [ModelType.MULTICLASS, ModelType.BINARY]:
             n_samples = label.groupby(label).count()
-            classes = test_dataset.classes
+            classes = train_dataset.classes
 
             results_array = []
             # Dict in format { Scorer : Dict { Class : Dict { Origin/Simple : score } } }
diff --git a/deepchecks/utils/metrics.py b/deepchecks/utils/metrics.py
--- a/deepchecks/utils/metrics.py
+++ b/deepchecks/utils/metrics.py
@@ -142,7 +142,7 @@ def score_perfect(self, dataset: 'base.Dataset'):
         """Calculate the perfect score of the current scorer for given dataset."""
         df = self.filter_nulls(dataset)
         perfect_model = PerfectModel()
-        perfect_model.fit(None, dataset.label_col)
+        perfect_model.fit(None, df[dataset.label_name])
         score = self._run_score(perfect_model, df, dataset)
         if isinstance(score, np.ndarray):
             # We expect the perfect score to be equal for all the classes, so takes the first one
diff --git a/deepchecks/utils/models.py b/deepchecks/utils/models.py
--- a/deepchecks/utils/models.py
+++ b/deepchecks/utils/models.py
@@ -37,7 +37,7 @@ def fit(self, X, y):  # pylint: disable=unused-argument,invalid-name
 
     def predict(self, X):  # pylint: disable=unused-argument,invalid-name
         """Predict on given X."""
-        return self.labels[X.index].to_numpy()
+        return self.labels.to_numpy()
 
     def predict_proba(self, X):  # pylint: disable=invalid-name
         """Predict proba for given X."""


Test patch:
Repository: deepchecks/deepchecks

Issue:
Wrong BSD license tag in the distribution
Although you clearly state in the repository that the software is licensed under AGPLv3, you do not explicitly specify the license type in the `setup.py` script. The license metadata is then derived from a wrong classifier: 
https://github.com/deepchecks/deepchecks/blob/7974e66d95358f4bccdd2ea16591b98e1ae74529/setup.py#L114

It's reflected in the [PyPI](https://pypi.org/project/deepchecks/) distribution:
![image](https://user-images.githubusercontent.com/36337649/148455950-abdd6865-9bf6-473a-88f4-3e86efa925a8.png)




Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -111,7 +111,7 @@
         'Intended Audience :: Science/Research',
         'Topic :: Software Development',
         'Topic :: Scientific/Engineering',
-        'License :: OSI Approved :: BSD License',
+        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',


Test patch:
Repository: deepchecks/deepchecks

Issue:
Permutation importance fails with nan labels
When we calculate permutation feature importance we need to filter out labels with nan values



Discussion:

Resolving patch:
diff --git a/deepchecks/utils/features.py b/deepchecks/utils/features.py
--- a/deepchecks/utils/features.py
+++ b/deepchecks/utils/features.py
@@ -194,7 +194,7 @@ def _calc_importance(
     n_repeats: int = 30,
     mask_high_variance_features: bool = False,
     random_state: int = 42,
-    n_samples: int = 10000,
+    n_samples: int = 10_000,
 ) -> pd.Series:
     """Calculate permutation feature importance. Return nonzero value only when std doesn't mask signal.
 
@@ -212,13 +212,12 @@ def _calc_importance(
     """
     dataset.validate_label()
 
-    n_samples = min(n_samples, dataset.n_samples)
-    dataset_sample_idx = dataset.label_col.sample(n_samples, random_state=random_state).index
-
-    scorer = get_scorer_single(model, dataset, multiclass_avg=False)
+    dataset_sample = dataset.sample(n_samples, drop_na_label=True, random_state=random_state)
 
+    # Test score time on the dataset sample
+    scorer = get_scorer_single(model, dataset)
     start_time = time.time()
-    scorer(model, dataset)
+    scorer(model, dataset_sample)
     calc_time = time.time() - start_time
 
     if calc_time * n_repeats * len(dataset.features) > _PERMUTATION_IMPORTANCE_TIMEOUT:
@@ -227,8 +226,8 @@ def _calc_importance(
 
     r = permutation_importance(
         model,
-        dataset.features_columns.loc[dataset_sample_idx, :],
-        dataset.label_col.loc[dataset_sample_idx],
+        dataset_sample.features_columns,
+        dataset_sample.label_col,
         n_repeats=n_repeats,
         random_state=random_state,
         n_jobs=-1


Test patch:
diff --git a/tests/base/feature_importance_utils_test.py b/tests/base/feature_importance_utils_test.py
--- a/tests/base/feature_importance_utils_test.py
+++ b/tests/base/feature_importance_utils_test.py
@@ -120,3 +120,17 @@ def test_no_warning_on_none_model(iris_dataset):
     # Assert
     assert_that(fi, none())
     assert_that(warn_record, has_length(0))
+
+
+def test_permutation_importance_with_nan_labels(iris_split_dataset_and_model):
+    # Arrange
+    train_ds, _, adaboost = iris_split_dataset_and_model
+    train_data = train_ds.data.copy()
+    train_data.loc[train_data['target'] != 2, 'target'] = None
+
+    # Act
+    feature_importances = calculate_feature_importance(adaboost, Dataset(train_data, label='target'),
+                                                       force_permutation=True)
+
+    # Assert
+    assert_that(feature_importances.sum(), close_to(1, 0.0001))
Repository: deepchecks/deepchecks

Issue:
[FEAT] Visualization of Regression Systematic Error box plot unclear
**Is your feature request related to a problem? Please describe.**
Visualization of Regression Systematic Error box plot unclear

![Screen Shot 2022-01-06 at 16 14 53](https://user-images.githubusercontent.com/26321553/148396690-7115e4e1-c81b-4458-81c7-f0d074db6990.png)

**Describe the solution you'd like**
Labels to be shorter and not sideways. Also labels are unclear.



Discussion:
Hi @JKL98ISR, can you be more specific about what is unclear about the labels? Other two comments I agree.
@yromanyshyn let's make the labels a bit shorter
@noamzbr @ItayGabbay @JKL98ISR 
is this expected (screenshot)?
![image](https://user-images.githubusercontent.com/71635444/148742564-a0a88b7e-9655-4648-bbc0-809926560e51.png)

Much better in my opinion @yromanyshyn 

Resolving patch:
diff --git a/deepchecks/checks/performance/regression_systematic_error.py b/deepchecks/checks/performance/regression_systematic_error.py
--- a/deepchecks/checks/performance/regression_systematic_error.py
+++ b/deepchecks/checks/performance/regression_systematic_error.py
@@ -51,14 +51,19 @@ def _regression_error_distribution(self, dataset: Dataset, model: BaseEstimator)
         diff = y_test - y_pred
         diff_mean = diff.mean()
 
-        fig = go.Figure()
-        fig.add_trace(go.Box(
-            x=diff,
-            name='Model prediction error',
-            boxmean=True  # represent mean
-        )).update_layout(
+        fig = (
+            go.Figure()
+            .add_trace(go.Box(
+                x=diff,
+                orientation='h',
+                name='Model prediction error',
+                hoverinfo='x',
+                boxmean=True))
+            .update_layout(
                 title_text='Box plot of the model prediction error',
-                width=800, height=500)
+                width=800,
+                height=500)
+        )
 
         display = [
             'Non-zero mean of the error distribution indicated the presents '


Test patch:
Repository: deepchecks/deepchecks

Issue:
[DOCS] Simple Model Comparison example improvement 
Currently, the Simple Model Comparison notebook lacks some explanations.
Please follow the guidelines from this issue: #543 to improve it


Discussion:

Resolving patch:
diff --git a/deepchecks/utils/validation.py b/deepchecks/utils/validation.py
--- a/deepchecks/utils/validation.py
+++ b/deepchecks/utils/validation.py
@@ -56,12 +56,8 @@ def validate_model(
 
     if isinstance(data, base.Dataset):
         features = data.features_columns
-        features_names = set(data.features)
     else:
         features = data
-        features_names = set(data.columns)
-
-    model_features = getattr(model, 'feature_names_in_', None)
 
     if features is None:
         raise errors.DeepchecksValueError(error_message.format(
@@ -73,16 +69,6 @@ def validate_model(
             'But function received empty dataset.'
         ))
 
-    try:
-        model_features = set(model_features)  # type: ignore
-        if model_features != features_names:
-            raise errors.DeepchecksValueError(error_message.format(
-                'But function received dataset with a different set of features.'
-            ))
-    except (TypeError, ValueError):
-        # in case if 'model.feature_names_in_' was None or not iterable
-        pass
-
     try:
         model.predict(features.head(1))
     except Exception as exc:


Test patch:
Repository: deepchecks/deepchecks

Issue:
[TYPO] Label ambiguity explanation
Current explanation: "Each row in the table shows an example of a data sample and the it's observed labels as a found in the dataset."


Discussion:

Resolving patch:
diff --git a/deepchecks/checks/integrity/label_ambiguity.py b/deepchecks/checks/integrity/label_ambiguity.py
--- a/deepchecks/checks/integrity/label_ambiguity.py
+++ b/deepchecks/checks/integrity/label_ambiguity.py
@@ -98,7 +98,7 @@ def run(self, dataset: Dataset, model=None) -> CheckResult:
         display = display.set_index(ambiguous_label_name)
 
         explanation = ('Each row in the table shows an example of a data sample '
-                       'and the it\'s observed labels as a found in the dataset.')
+                       'and the its observed labels as found in the dataset.')
 
         display = None if display.empty else [explanation, display.head(self.n_to_show)]
 


Test patch:
Repository: deepchecks/deepchecks

Issue:
[FEAT] Make notebook work better in colab (currently links nor widgets work there)



Discussion:
maybe best option is to open the HTML output we have in a new tab from a quick google I think it's possible.
 @shir22 @ItayGabbay thoughts?
I think it's gonna be a bit annoying. But also I saw a lot of libraries that are doing that, because colab's ui is tricky. If it's a fast solution I'd like to try it first before we are releasing it to public.

Resolving patch:
diff --git a/deepchecks/base/display_suite.py b/deepchecks/base/display_suite.py
--- a/deepchecks/base/display_suite.py
+++ b/deepchecks/base/display_suite.py
@@ -229,7 +229,8 @@ def _display_suite_no_widgets(summary: str,
         others_df = dataframe_to_html(others_table.style.hide_index())
         display_html(others_h2 + others_df, raw=True)
 
-    display_html(f'<br><a href="#summary_{unique_id}" style="font-size: 14px">Go to top</a>', raw=True)
+    if unique_id:
+        display_html(f'<br><a href="#summary_{unique_id}" style="font-size: 14px">Go to top</a>', raw=True)
 
 
 def display_suite_result(suite_name: str, results: List[Union[CheckResult, CheckFailure]],
diff --git a/deepchecks/base/suite.py b/deepchecks/base/suite.py
--- a/deepchecks/base/suite.py
+++ b/deepchecks/base/suite.py
@@ -11,9 +11,12 @@
 """Module containing the Suite object, used for running a set of checks together."""
 # pylint: disable=broad-except
 import abc
+import io
 from collections import OrderedDict
 from typing import Union, List, Optional, Tuple, Any, Container, Mapping
 
+from IPython.core.display import display_html
+from IPython.core.getipython import get_ipython
 import jsonpickle
 
 from deepchecks.base.display_suite import display_suite_result, ProgressBar
@@ -43,7 +46,13 @@ def __repr__(self):
         return self.name
 
     def _ipython_display_(self):
-        display_suite_result(self.name, self.results)
+        # google colab has no support for widgets but good support for viewing html pages in the output
+        if 'google.colab' in str(get_ipython()):
+            html_out = io.StringIO()
+            display_suite_result(self.name, self.results, html_out=html_out)
+            display_html(html_out.getvalue(), raw=True)
+        else:
+            display_suite_result(self.name, self.results)
 
     def show(self):
         """Display suite result."""


Test patch:
Repository: deepchecks/deepchecks

Issue:
[BUG] resources/suite_output.html file is missing when installing not via git
**Describe the bug**
can't use save_as_html because suite_output.html file is missing

**To Reproduce**
pip install deepchecks
suite_result.save_as_html()

**Expected behavior**
save as html

**Environment (please complete the following information):**
 - OS: linux
 - Python Version: 3.7
 - Deepchecks Version: 0.3.1



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -74,7 +74,6 @@
 
 import setuptools
 from setuptools import setup
-from distutils.util import convert_path
 import os
 
 main_ns = {}
@@ -89,9 +88,6 @@
     with open(requirementPath) as f:
         install_requires = f.read().splitlines()
 
-
-
-
 setup(
     name='deepchecks',
     version=VER,


Test patch:
Repository: deepchecks/deepchecks

Issue:
[DOCS] Improve label ambiguity notebook
Follow the guidelines on #543


Discussion:

Resolving patch:
diff --git a/deepchecks/base/dataset.py b/deepchecks/base/dataset.py
--- a/deepchecks/base/dataset.py
+++ b/deepchecks/base/dataset.py
@@ -765,7 +765,8 @@ def columns_info(self) -> t.Dict[Hashable, str]:
     def select(
             self: TDataset,
             columns: t.Union[Hashable, t.List[Hashable], None] = None,
-            ignore_columns: t.Union[Hashable, t.List[Hashable], None] = None
+            ignore_columns: t.Union[Hashable, t.List[Hashable], None] = None,
+            keep_label: bool = False
     ) -> TDataset:
         """Filter dataset columns by given params.
 
@@ -786,6 +787,9 @@ def select(
         DeepchecksValueError
             In case one of columns given don't exists raise error
         """
+        if keep_label and columns and self.label_name not in columns:
+            columns.append(self.label_name)
+
         new_data = select_from_dataframe(self._data, columns, ignore_columns)
         if new_data.equals(self.data):
             return self
diff --git a/deepchecks/checks/integrity/label_ambiguity.py b/deepchecks/checks/integrity/label_ambiguity.py
--- a/deepchecks/checks/integrity/label_ambiguity.py
+++ b/deepchecks/checks/integrity/label_ambiguity.py
@@ -63,9 +63,9 @@ def run_logic(self, context: CheckRunContext, dataset_type: str = 'train') -> Ch
 
         context.assert_classification_task()
 
-        dataset = dataset.select(self.columns, self.ignore_columns)
+        dataset = dataset.select(self.columns, self.ignore_columns, keep_label=True)
 
-        label_col = dataset.label_name
+        label_col = context.label_name
 
         # HACK: pandas have bug with groupby on category dtypes, so until it fixed, change dtypes manually
         df = dataset.data
@@ -87,7 +87,7 @@ def run_logic(self, context: CheckRunContext, dataset_type: str = 'train') -> Ch
 
             group_df = group_data[1]
             sample_values = dict(group_df[dataset.features].iloc[0])
-            labels = tuple(group_df[label_col].unique())
+            labels = tuple(sorted(group_df[label_col].unique()))
             n_data_sample = group_df.shape[0]
             num_ambiguous += n_data_sample
 
@@ -96,7 +96,8 @@ def run_logic(self, context: CheckRunContext, dataset_type: str = 'train') -> Ch
         display = display.set_index(ambiguous_label_name)
 
         explanation = ('Each row in the table shows an example of a data sample '
-                       'and the its observed labels as found in the dataset.')
+                       'and the its observed labels as found in the dataset. '
+                       f'Showing top {self.n_to_show} of {display.shape[0]}')
 
         display = None if display.empty else [explanation, display.head(self.n_to_show)]
 


Test patch: