Repository: deepchecks/deepchecks

Issue:
BUG:New Label train test in regression task
When running the full suite on a regression task, I encouner 2 differnet bugs:

1. This check is appearing twice
2. This check is appearing (In regression i don't expect it to be there)
![image](https://user-images.githubusercontent.com/20860465/147875467-c2def489-b41e-4dda-8d3f-961fbffb9964.png)



Discussion:

Resolving patch:
diff --git a/deepchecks/checks/integrity/label_ambiguity.py b/deepchecks/checks/integrity/label_ambiguity.py
--- a/deepchecks/checks/integrity/label_ambiguity.py
+++ b/deepchecks/checks/integrity/label_ambiguity.py
@@ -16,6 +16,7 @@
 from deepchecks import Dataset, ConditionResult
 from deepchecks.base.check import CheckResult, SingleDatasetBaseCheck
 from deepchecks.errors import DeepchecksValueError
+from deepchecks.utils.metrics import task_type_validation, ModelType
 from deepchecks.utils.strings import format_percent
 from deepchecks.utils.typing import Hashable
 
@@ -52,16 +53,18 @@ def run(self, dataset: Dataset, model=None) -> CheckResult:
         """Run check.
 
         Args:
-          dataset(Dataset): any dataset.
+            dataset(Dataset): any dataset.
+            model (any): used to check task type (default: None)
 
         Returns:
-          (CheckResult): percentage of ambiguous samples and display of the top n_to_show most ambiguous.
+            (CheckResult): percentage of ambiguous samples and display of the top n_to_show most ambiguous.
         """
         dataset: Dataset = Dataset.validate_dataset(dataset)
         dataset = dataset.select(self.columns, self.ignore_columns)
 
-        if (model and hasattr(model, 'is_regression') and model.is_regression()) or \
-                (dataset.label_type == 'regression_label'):
+        if model:
+            task_type_validation(model, dataset, [ModelType.MULTICLASS, ModelType.BINARY])
+        elif dataset.label_type == 'regression_label':
             raise DeepchecksValueError('Task type cannot be regression')
 
         label_col = dataset.label_name
diff --git a/deepchecks/checks/integrity/new_label.py b/deepchecks/checks/integrity/new_label.py
--- a/deepchecks/checks/integrity/new_label.py
+++ b/deepchecks/checks/integrity/new_label.py
@@ -13,6 +13,8 @@
 
 from deepchecks import Dataset
 from deepchecks.base.check import CheckResult, TrainTestBaseCheck, ConditionResult
+from deepchecks.errors import DeepchecksValueError
+from deepchecks.utils.metrics import task_type_validation, ModelType
 from deepchecks.utils.strings import format_percent
 
 import pandas as pd
@@ -32,7 +34,7 @@ def run(self, train_dataset: Dataset, test_dataset: Dataset, model=None) -> Chec
         Args:
             train_dataset (Dataset): The training dataset object.
             test_dataset (Dataset): The test dataset object.
-            model: any = None - not used in the check
+            model (any): used to check task type (default: None)
 
         Returns:
             CheckResult: value is a dictionary that shows label column with new labels
@@ -42,15 +44,21 @@ def run(self, train_dataset: Dataset, test_dataset: Dataset, model=None) -> Chec
             DeepchecksValueError: If the datasets are not a Dataset instance or do not contain label column
         """
         return self._new_label_train_test(train_dataset=train_dataset,
-                                          test_dataset=test_dataset)
+                                          test_dataset=test_dataset,
+                                          model=model)
 
-    def _new_label_train_test(self, train_dataset: Dataset, test_dataset: Dataset):
-        test_dataset = Dataset.validate_dataset_or_dataframe(test_dataset)
-        train_dataset = Dataset.validate_dataset_or_dataframe(train_dataset)
+    def _new_label_train_test(self, train_dataset: Dataset, test_dataset: Dataset, model: any = None):
+        test_dataset = Dataset.validate_dataset(test_dataset)
+        train_dataset = Dataset.validate_dataset(train_dataset)
         test_dataset.validate_label()
         train_dataset.validate_label()
         test_dataset.validate_shared_label(train_dataset)
 
+        if model:
+            task_type_validation(model, train_dataset, [ModelType.MULTICLASS, ModelType.BINARY])
+        elif train_dataset.label_type == 'regression_label':
+            raise DeepchecksValueError('Task type cannot be regression')
+
         label_column = train_dataset.validate_shared_label(test_dataset)
 
         n_test_samples = test_dataset.n_samples
diff --git a/deepchecks/suites/default_suites.py b/deepchecks/suites/default_suites.py
--- a/deepchecks/suites/default_suites.py
+++ b/deepchecks/suites/default_suites.py
@@ -73,7 +73,7 @@ def train_test_validation() -> Suite:
         WholeDatasetDrift().add_condition_overall_drift_value_not_greater_than(),
         DominantFrequencyChange().add_condition_ratio_of_change_not_more_than(),
         CategoryMismatchTrainTest().add_condition_new_category_ratio_not_greater_than(),
-        NewLabelTrainTest().add_condition_new_label_ratio_not_greater_than(),
+        NewLabelTrainTest().add_condition_new_labels_not_greater_than(),
         StringMismatchComparison().add_condition_no_new_variants(),
         DatasetsSizeComparison().add_condition_test_train_size_ratio_not_smaller_than(),
         train_test_leakage()
@@ -95,7 +95,6 @@ def model_evaluation() -> Suite:
         SimpleModelComparison().add_condition_ratio_not_less_than(),
         CalibrationScore(),
         TrustScoreComparison().add_condition_mean_score_percent_decline_not_greater_than(),
-        NewLabelTrainTest().add_condition_new_labels_not_greater_than(),
         RegressionSystematicError().add_condition_systematic_error_ratio_to_rmse_not_greater_than(),
         RegressionErrorDistribution().add_condition_kurtosis_not_less_than(),
         BoostingOverfit().add_condition_test_score_percent_decline_not_greater_than(),


Test patch:
diff --git a/tests/checks/integrity/new_label_test.py b/tests/checks/integrity/new_label_test.py
--- a/tests/checks/integrity/new_label_test.py
+++ b/tests/checks/integrity/new_label_test.py
@@ -25,14 +25,16 @@ def test_dataset_wrong_input():
     # Act & Assert
     assert_that(calling(NewLabelTrainTest().run).with_args(x, x),
                 raises(DeepchecksValueError,
-                'dataset must be of type DataFrame or Dataset. instead got: str'))
+                'Check requires dataset to be of type Dataset. instead got: str'))
 
 
 def test_no_new_label():
     train_data = {'col1': [1, 2, 3]}
     test_data = {'col1': [1, 1, 2, 3]}
-    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1']), label_name='col1')
-    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1']), label_name='col1')
+    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1']),
+                            label_name='col1', label_type="classification_label")
+    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1']),
+                           label_name='col1', label_type="classification_label")
 
     # Arrange
     check = NewLabelTrainTest()
@@ -45,8 +47,10 @@ def test_no_new_label():
 def test_new_label():
     train_data = {'col1': [1, 2, 3]}
     test_data = {'col1': [1, 2, 3, 4]}
-    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1']), label_name='col1')
-    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1']), label_name='col1')
+    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1']),
+                            label_name='col1', label_type="classification_label")
+    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1']),
+                           label_name='col1', label_type="classification_label")
 
     # Arrange
     check = NewLabelTrainTest()
@@ -62,8 +66,10 @@ def test_new_label():
 def test_missing_label():
     train_data = {'col1': [1, 2, 3, 4]}
     test_data = {'col1': [1, 2, 3]}
-    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1']), label_name='col1')
-    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1']), label_name='col1')
+    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1']),
+                            label_name='col1', label_type="classification_label")
+    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1']),
+                           label_name='col1', label_type="classification_label")
 
     # Arrange
     check = NewLabelTrainTest()
@@ -76,8 +82,10 @@ def test_missing_label():
 def test_missing_new_label():
     train_data = {'col1': [1, 2, 3, 4]}
     test_data = {'col1': [1, 2, 3, 5]}
-    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1']), label_name='col1')
-    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1']), label_name='col1')
+    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1']),
+                            label_name='col1', label_type="classification_label")
+    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1']),
+                           label_name='col1', label_type="classification_label")
 
     # Arrange
     check = NewLabelTrainTest()
@@ -93,8 +101,10 @@ def test_missing_new_label():
 def test_multiple_categories():
     train_data = {'col1': [1, 2, 3, 4], 'col2': [1, 2, 3, 4]}
     test_data = {'col1': [1, 2, 3, 5], 'col2': [1, 2, 3, 4]}
-    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']), label_name='col1')
-    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']), label_name='col1')
+    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']),
+                            label_name='col1', label_type="classification_label")
+    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']),
+                           label_name='col1', label_type="classification_label")
 
     # Arrange
     check = NewLabelTrainTest()
@@ -110,8 +120,10 @@ def test_multiple_categories():
 def test_condition_number_of_new_labels_pass():
     train_data = {'col1': [1, 2, 3, 4], 'col2': [1, 2, 3, 4]}
     test_data = {'col1': [1, 2, 3, 5], 'col2': [1, 2, 3, 4]}
-    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']), label_name='col1')
-    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']), label_name='col1')
+    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']),
+                            label_name='col1', label_type="classification_label")
+    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']),
+                           label_name='col1', label_type="classification_label")
 
     # Arrange
     check = NewLabelTrainTest().add_condition_new_labels_not_greater_than(3)
@@ -128,8 +140,10 @@ def test_condition_number_of_new_labels_pass():
 def test_condition_number_of_new_labels_fail():
     train_data = {'col1': [1, 2, 3, 4], 'col2': [1, 2, 3, 4]}
     test_data = {'col1': [1, 2, 3, 5], 'col2': [1, 2, 3, 4]}
-    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']), label_name='col1')
-    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']), label_name='col1')
+    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']),
+                            label_name='col1', label_type="classification_label")
+    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']),
+                           label_name='col1', label_type="classification_label")
 
     # Arrange
     check = NewLabelTrainTest().add_condition_new_labels_not_greater_than(0)
@@ -147,8 +161,10 @@ def test_condition_number_of_new_labels_fail():
 def test_condition_ratio_of_new_label_samples_pass():
     train_data = {'col1': [1, 2, 3, 4], 'col2': [1, 2, 3, 4]}
     test_data = {'col1': [1, 2, 3, 5], 'col2': [1, 2, 3, 4]}
-    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']), label_name='col1')
-    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']), label_name='col1')
+    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']),
+                            label_name='col1', label_type="classification_label")
+    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']),
+                           label_name='col1', label_type="classification_label")
 
     # Arrange
     check = NewLabelTrainTest().add_condition_new_label_ratio_not_greater_than(0.3)
@@ -165,8 +181,10 @@ def test_condition_ratio_of_new_label_samples_pass():
 def test_condition_ratio_of_new_label_samples_fail():
     train_data = {'col1': [1, 2, 3, 4], 'col2': [1, 2, 3, 4]}
     test_data = {'col1': [1, 2, 3, 5], 'col2': [1, 2, 3, 4]}
-    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']), label_name='col1')
-    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']), label_name='col1')
+    train_dataset = Dataset(pd.DataFrame(data=train_data, columns=['col1', 'col2']),
+                            label_name='col1', label_type="classification_label")
+    test_dataset = Dataset(pd.DataFrame(data=test_data, columns=['col1', 'col2']),
+                           label_name='col1', label_type="classification_label")
 
     # Arrange
     check = NewLabelTrainTest().add_condition_new_label_ratio_not_greater_than(0.1)