internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: deepchecks/deepchecks

Issue:
[DOCS][CV]Documentation placeholders



Discussion:

Resolving patch:
diff --git a/deepchecks/vision/base.py b/deepchecks/vision/base.py
--- a/deepchecks/vision/base.py
+++ b/deepchecks/vision/base.py
@@ -233,7 +233,7 @@ def assert_regression_task(self):
 
 
 class Check(BaseCheck):
-    """Base class for all tabular checks."""
+    """Base class for all computer vision checks."""
 
     def __init__(self):
         # pylint: disable=super-init-not-called
diff --git a/docs/source/conf.py b/docs/source/conf.py
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -487,14 +487,21 @@ def get_check_example_api_reference(filepath: str) -> t.Optional[str]:
             .replace(".ipynb", "")
             .replace(".py", "")
     )
+    type = filepath.split("/")[2]
 
     import deepchecks.tabular.checks
-    check_clazz = getattr(deepchecks.tabular.checks, notebook_name, None)
+    import deepchecks.vision.checks
+    tabular_check_clazz = getattr(deepchecks.tabular.checks, notebook_name, None)
+    vision_check_clazz = getattr(deepchecks.vision.checks, notebook_name, None)
 
-    if check_clazz is None or not hasattr(check_clazz, "__module__"):
+    mapping = {
+        'tabular': tabular_check_clazz,
+        'vision': vision_check_clazz
+    }
+    if type not in mapping or mapping[type] is None or not hasattr(mapping[type], "__module__"):
         return
 
-    clazz_module = ".".join(check_clazz.__module__.split("."))
+    clazz_module = ".".join(mapping[type].__module__.split("."))
 
     apipath = f"/api/generated/{clazz_module}.{notebook_name}"
     result = f"* :doc:`API Reference - {notebook_name} <{apipath}>`"


Test patch:
diff --git a/docs/source/examples/checks/vision/checks/distribution/train_test_label_drift.ipynb b/docs/source/examples/checks/vision/distribution/train_test_label_drift.ipynb
similarity index 100%
rename from docs/source/examples/checks/vision/checks/distribution/train_test_label_drift.ipynb
rename to docs/source/examples/checks/vision/distribution/train_test_label_drift.ipynb
Repository: deepchecks/deepchecks

Issue:
[FEAT][GENERAL] check for the latest version and notify if using an older version
**Describe the solution you'd like**
Check that the latest version is installed and raise a warning when importing the package if it's not the latest version.



Discussion:

Resolving patch:
diff --git a/deepchecks/__init__.py b/deepchecks/__init__.py
--- a/deepchecks/__init__.py
+++ b/deepchecks/__init__.py
@@ -11,10 +11,11 @@
 """Deepchecks."""
 import os
 import pathlib
+import http.client
 import warnings
 import matplotlib
 import plotly.io as pio
-
+from pkg_resources import parse_version
 
 from deepchecks.utils.ipython import is_notebook
 from deepchecks.tabular import (
@@ -95,6 +96,20 @@
     MODULE_DIR = pathlib.Path(__file__).absolute().parent.parent
     with open(os.path.join(MODULE_DIR, 'VERSION'), 'r', encoding='utf-8') as f:
         __version__ = f.read().strip()
-except Exception:  # pylint: disable=broad-except
+except:  # pylint: disable=bare-except # noqa
     # If version file can't be found, leave version empty
     __version__ = ''
+
+# Check for latest version
+try:
+    disable = os.environ.get('DEEPCHECKS_DISABLE_LATEST', 'false').lower() == 'true'
+    if not disable:
+        conn = http.client.HTTPSConnection('api.deepchecks.com', timeout=3)
+        conn.request('GET', '/latest')
+        response = conn.getresponse()
+        latest_version = response.read().decode('utf-8')
+        if __version__ and parse_version(__version__) < parse_version(latest_version):
+            warnings.warn('Looks like you are using outdated version of deepchecks. consider upgrading using'
+                          ' pip install -U deepchecks')
+except:  # pylint: disable=bare-except # noqa
+    pass


Test patch:
Repository: deepchecks/deepchecks

Issue:
[FEAT][CV] Group train and test check results together for SingleDatasetCheck
Currently the vision suite first displays all results for train and then all results for test.

DoD:
Results for same check for train and test should show one after the other


Discussion:

Resolving patch:
diff --git a/deepchecks/vision/base.py b/deepchecks/vision/base.py
--- a/deepchecks/vision/base.py
+++ b/deepchecks/vision/base.py
@@ -414,6 +414,8 @@ def run(
                     checks[str(check_idx) + ' - Train'] = check
                 if test_dataset is not None:
                     checks[str(check_idx) + ' - Test'] = check
+            else:
+                raise DeepchecksNotSupportedError(f'Don\'t know to handle check type {type(check)}')
 
         run_train_test_checks = train_dataset is not None and test_dataset is not None
 
@@ -463,14 +465,18 @@ def run(
         for check_idx, result in results.items():
             if isinstance(result, CheckResult):
                 result = finalize_check_result(result, checks[check_idx])
-                result.header = (
-                    f'{result.get_header()} - Train Dataset'
-                    if str(check_idx).endswith(' - Train')
-                    else f'{result.get_header()} - Test Dataset'
-                )
                 results[check_idx] = result
-
-        return SuiteResult(self.name, list(results.values()))
+                # Update header only if both train and test ran
+                if run_train_test_checks:
+                    result.header = (
+                        f'{result.get_header()} - Train Dataset'
+                        if str(check_idx).endswith(' - Train')
+                        else f'{result.get_header()} - Test Dataset'
+                    )
+
+        # The results are ordered as they ran instead of in the order they were defined, therefore sort by key
+        sorted_result_values = [value for name, value in sorted(results.items(), key=lambda pair: str(pair[0]))]
+        return SuiteResult(self.name, sorted_result_values)
 
     def _update_loop(
         self,
diff --git a/deepchecks/vision/checks/__init__.py b/deepchecks/vision/checks/__init__.py
--- a/deepchecks/vision/checks/__init__.py
+++ b/deepchecks/vision/checks/__init__.py
@@ -10,7 +10,7 @@
 #
 """Module importing all vision checks."""
 from .performance import ClassPerformance, MeanAveragePrecisionReport, MeanAverageRecallReport, \
-                         RobustnessReport, ConfusionMatrixReport, SimpleModelComparison
+                         RobustnessReport, ConfusionMatrixReport, SimpleModelComparison, ImageSegmentPerformance
 from .distribution import TrainTestLabelDrift, ImageDatasetDrift, ImagePropertyDrift, TrainTestPredictionDrift
 
 __all__ = [
@@ -23,5 +23,6 @@
     'TrainTestLabelDrift',
     'ImageDatasetDrift',
     'ImagePropertyDrift',
-    'TrainTestPredictionDrift'
+    'TrainTestPredictionDrift',
+    'ImageSegmentPerformance'
 ]


Test patch:
Repository: deepchecks/deepchecks

Issue:
[BUG][CV] Confusion matrix display labels are inverted
The "True value" and "Predicted value" are inverted. The matrix should be transposed prior to display. 



Discussion:

Resolving patch:
diff --git a/deepchecks/vision/checks/performance/confusion_matrix.py b/deepchecks/vision/checks/performance/confusion_matrix.py
--- a/deepchecks/vision/checks/performance/confusion_matrix.py
+++ b/deepchecks/vision/checks/performance/confusion_matrix.py
@@ -145,7 +145,7 @@ def update_object_detection(self, predictions, labels):
                 # detections are empty, update matrix for labels
                 for label in image_labels:
                     gt_class = int(label[0].item())
-                    self.matrix[self.num_classes, gt_class] += 1
+                    self.matrix[gt_class, self.num_classes] += 1
                 continue
 
             all_ious = np.zeros((len(image_labels), len(detections_passed_threshold)))
@@ -174,18 +174,18 @@ def update_object_detection(self, predictions, labels):
                 gt_class = int(label[0])
                 if all_matches.shape[0] > 0 and all_matches[all_matches[:, 0] == i].shape[0] == 1:
                     detection_class = int(image_detections[int(all_matches[all_matches[:, 0] == i, 1][0])][5])
-                    self.matrix[detection_class, gt_class] += 1
+                    self.matrix[gt_class, detection_class] += 1
                 else:
-                    self.matrix[self.num_classes, gt_class] += 1
+                    self.matrix[gt_class, self.num_classes] += 1
 
             for i, detection in enumerate(image_detections):
                 if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0:
                     detection_class = int(detection[5])
-                    self.matrix[detection_class, self.num_classes] += 1
+                    self.matrix[self.num_classes, detection_class] += 1
 
     def update_classification(self, predictions, labels):
         """Update the confusion matrix by batch for classification task."""
         for predicted_classes, image_labels in zip(predictions, labels):
             detected_class = max(range(len(predicted_classes)), key=predicted_classes.__getitem__)
 
-            self.matrix[detected_class, image_labels] += 1
+            self.matrix[image_labels, detected_class] += 1


Test patch:
Repository: deepchecks/deepchecks

Issue:
[BUG] [CV] heatmap comparison error
```
error: OpenCV(4.5.5) /Users/runner/work/opencv-python/opencv-python/opencv/modules/imgproc/src/color.simd_helpers.hpp:94: error: (-2:Unspecified error) in function 'cv::impl::(anonymous namespace)::CvtHelper, cv::impl::(anonymous namespace)::Set<1, -1, -1>, cv::impl::(anonymous namespace)::Set<0, 2, 5>, cv::impl::(anonymous namespace)::NONE>::CvtHelper(cv::InputArray, cv::OutputArray, int) [VScn = cv::impl::(anonymous namespace)::Set<3, 4, -1>, VDcn = cv::impl::(anonymous namespace)::Set<1, -1, -1>, VDepth = cv::impl::(anonymous namespace)::Set<0, 2, 5>, sizePolicy = cv::impl::(anonymous namespace)::NONE]'
> Unsupported depth of input image:
>     'VDepth::contains(depth)'
> where
>     'depth' is 6 (CV_64F)
```


Discussion:

Resolving patch:
diff --git a/deepchecks/vision/checks/distribution/heatmap_comparison.py b/deepchecks/vision/checks/distribution/heatmap_comparison.py
--- a/deepchecks/vision/checks/distribution/heatmap_comparison.py
+++ b/deepchecks/vision/checks/distribution/heatmap_comparison.py
@@ -258,7 +258,7 @@ def _grayscale_sum_image(batch: Iterable[np.ndarray], target_shape: List[Tuple[i
             if img.shape[2] == 1:
                 resized_img = img
             elif img.shape[2] == 3:
-                resized_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+                resized_img = cv2.cvtColor(img.astype('float32'), cv2.COLOR_RGB2GRAY)
             else:
                 raise NotImplementedError('Images must be RGB or grayscale')
 
diff --git a/deepchecks/vision/vision_data.py b/deepchecks/vision/vision_data.py
--- a/deepchecks/vision/vision_data.py
+++ b/deepchecks/vision/vision_data.py
@@ -129,8 +129,17 @@ def batch_to_images(self, batch) -> Iterable[np.ndarray]:
 
         Examples
         --------
-        >>> def batch_to_images(self, batch):
-        ...     return batch[0]
+        >>> import numpy as np
+        ...
+        ...
+        ... def batch_to_images(self, batch):
+        ...     # Converts a batch of normalized images to rgb images with range [0, 255]
+        ...     inp = batch[0].detach().numpy().transpose((0, 2, 3, 1))
+        ...     mean = [0.485, 0.456, 0.406]
+        ...     std = [0.229, 0.224, 0.225]
+        ...     inp = std * inp + mean
+        ...     inp = np.clip(inp, 0, 1)
+        ...     return inp * 255
 
         Notes
         -----


Test patch:
Repository: deepchecks/deepchecks

Issue:
[BUG][CV] Making sure image data is un-normalized 
**Describe the bug**
Currently, if image data returned by image_to_batch is normalized, the validate_data() method prints "The data inside the iterable appear to be normalized.".
This is not clear because it's not clear what is normalized and that it's a bad thing. 

**Expected behavior**
Message should make it clear what was found (what data range) and what is expected (0-255)

**Screenshots**
![image](https://user-images.githubusercontent.com/17730502/157264199-134e23c8-501b-45fc-a3e8-bb5f979020e3.png)




Discussion:

Resolving patch:
diff --git a/deepchecks/vision/vision_data.py b/deepchecks/vision/vision_data.py
--- a/deepchecks/vision/vision_data.py
+++ b/deepchecks/vision/vision_data.py
@@ -304,10 +304,11 @@ def validate_image_data(self, batch):
             raise ValidationError('The data inside the iterable must be a 3D array.')
         if sample.shape[2] not in [1, 3]:
             raise ValidationError('The data inside the iterable must have 1 or 3 channels.')
-        if sample.min() < 0 or sample.max() > 255:
-            raise ValidationError('The data inside the iterable must be in the range [0, 255].')
-        if np.all(sample <= 1):
-            raise ValidationError('The data inside the iterable appear to be normalized.')
+        sample_min = sample.min()
+        sample_max = sample.max()
+        if sample_min < 0 or sample_max > 255 or sample_max <= 1:
+            raise ValidationError(f'Image data found to be in range [{sample_min}, {sample_max}] instead of expected '
+                                  f'range [0, 255].')
 
     def _get_samples_per_class(self):
         """


Test patch:
diff --git a/tests/vision/utils_tests/formatters_test.py b/tests/vision/utils_tests/formatters_test.py
--- a/tests/vision/utils_tests/formatters_test.py
+++ b/tests/vision/utils_tests/formatters_test.py
@@ -84,14 +84,23 @@ def test_data_formatter_wrong_color_channel():
     )
 
 
-def test_data_formatter_invalid_values():
+def test_data_formatter_large_values():
     class BadImage(VisionData):
         def batch_to_images(self, batch):
             return batch * 300
     batch = next(iter(numpy_shape_dataloader((10, 10, 3))))
     assert_that(
         calling(BadImage(numpy_shape_dataloader((10, 10, 3))).validate_image_data).with_args(batch),
-        raises(ValidationError, r'The data inside the iterable must be in the range \[0, 255\].')
+        raises(ValidationError, r'Image data found to be in range \[76500.0, 76500.0\] instead of expected range '
+                                r'\[0, 255\].')
+    )
+
+
+def test_data_formatter_normalized_data():
+    batch = next(iter(numpy_shape_dataloader((10, 10, 3), value=1)))
+    assert_that(
+        calling(SimpleImageData(numpy_shape_dataloader((10, 10, 3))).validate_image_data).with_args(batch),
+        raises(ValidationError, r'Image data found to be in range \[1.0, 1.0\] instead of expected range \[0, 255\].')
     )
Repository: deepchecks/deepchecks

Issue:
[FEAT][DOCS] New VisionData user guide



Discussion:

Resolving patch:
diff --git a/deepchecks/vision/vision_data.py b/deepchecks/vision/vision_data.py
--- a/deepchecks/vision/vision_data.py
+++ b/deepchecks/vision/vision_data.py
@@ -356,6 +356,18 @@ def validate_image_data(self, batch):
             raise ValidationError(f'Image data found to be in range [{sample_min}, {sample_max}] instead of expected '
                                   f'range [0, 255].')
 
+    def validate_format(self, model):
+        """Validate the correctness of the data class implementation according to the expected format.
+
+        Parameters
+        ----------
+        model : Model
+            Model to validate the data class implementation against.
+
+        """
+        from deepchecks.vision.utils.validation import validate_extractors  # pylint: disable=import-outside-toplevel
+        validate_extractors(self, model)
+
     def __iter__(self):
         """Return an iterator over the dataset."""
         return iter(self._data_loader)


Test patch:
diff --git a/tests/vision/base/test_vision_data.py b/tests/vision/base/test_vision_data.py
--- a/tests/vision/base/test_vision_data.py
+++ b/tests/vision/base/test_vision_data.py
@@ -22,7 +22,7 @@
     has_entries,
     instance_of,
     all_of,
-    contains_exactly
+    contains_exactly, not_
 )
 import albumentations as A
 import imgaug.augmenters as iaa
@@ -291,3 +291,10 @@ def test_data_at_batch_of_index(mnist_dataset_train):
     # Assert
     assert torch.equal(batch[0], single_batch[0])
     assert torch.equal(batch[1], single_batch[1])
+
+
+def test_validate_format(mnist_dataset_train, trained_mnist):
+
+    # Act & Assert
+    assert_that(calling(mnist_dataset_train.validate_format).with_args(trained_mnist),
+                not_(raises(DeepchecksValueError)))
\ No newline at end of file
Repository: deepchecks/deepchecks

Issue:
[BUG][CV] deepchecks should either validate model.eval status or actively create it 
**Describe the bug**
Checks can crash or give wrong results if models are set by mistake to training mode. 

**To Reproduce**
Run metrics check with model.train() preceding it.

**Expected behavior**
Either we validate that the model is in the correct state, or we actively call model.eval()



Discussion:

Resolving patch:
diff --git a/deepchecks/vision/context.py b/deepchecks/vision/context.py
--- a/deepchecks/vision/context.py
+++ b/deepchecks/vision/context.py
@@ -122,6 +122,13 @@ def __init__(self,
         self._device = torch.device(device) if isinstance(device, str) else (device if device else torch.device('cpu'))
 
         if model is not None:
+            if not isinstance(model, nn.Module):
+                logger.warning('Model is not a torch.nn.Module. Deepchecks can\'t validate that model is in '
+                               'evaluation state.')
+            else:
+                if model.training:
+                    raise DatasetValidationError('Model is not in evaluation state. Please set model training '
+                                                 'parameter to False or run model.eval() before passing it.')
             for dataset, dataset_type in zip([train, test], ['train', 'test']):
                 if dataset is not None:
                     try:


Test patch:
diff --git a/tests/vision/base/test_context.py b/tests/vision/base/test_context.py
--- a/tests/vision/base/test_context.py
+++ b/tests/vision/base/test_context.py
@@ -9,6 +9,8 @@
 # ----------------------------------------------------------------------------
 #
 #
+import copy
+
 import torch
 from torch import nn
 from hamcrest import (
@@ -137,13 +139,26 @@ def test_context_initialization_with_model_only(trained_mnist):
     Context(model_name="Name", model=trained_mnist)
 
 
+def test_context_initialization_with_training_model(trained_mnist):
+    trained_mnist = copy.deepcopy(trained_mnist)
+    trained_mnist.train()
+    assert_that(
+        calling(Context).with_args(model_name="Name", model=trained_mnist),
+        raises(
+            DatasetValidationError,
+            r'Model is not in evaluation state. Please set model training '
+            r'parameter to False or run model.eval\(\) before passing it.')
+    )
+
+
 def test_context_initialization_with_broken_model(mnist_dataset_train, mnist_dataset_test):
     # Arrange
     class BrokenModel(nn.Module):
         def __call__(self, *args, **kwargs):
-            raise Exception("Unvalid arguments")
+            raise Exception("Invalid arguments")
 
     model = BrokenModel()
+    model.eval()
 
     # Act & Assert
     assert_that(
@@ -153,5 +168,5 @@ def __call__(self, *args, **kwargs):
                             model=model),
         raises(
             Exception,
-            r'Unvalid arguments')
+            r'Invalid arguments')
     )
Repository: deepchecks/deepchecks

Issue:
[BUG] Refactor all imports to use `deepchecks.tabular` instead of `deepchecks.`
**Describe the bug**

After our refactor of the project structure in order to support both tabular and vision, we are left we old imports in the files that still points to `deepchecks.` instead of `deepchecks.tabular.`. This works for now but causes a warning to appear.


Discussion:

Resolving patch:
diff --git a/deepchecks/core/check_utils/whole_dataset_drift_utils.py b/deepchecks/core/check_utils/whole_dataset_drift_utils.py
--- a/deepchecks/core/check_utils/whole_dataset_drift_utils.py
+++ b/deepchecks/core/check_utils/whole_dataset_drift_utils.py
@@ -16,7 +16,7 @@
 import numpy as np
 import pandas as pd
 
-from deepchecks import Dataset  # TODO: Remove?
+from deepchecks.tabular import Dataset
 from deepchecks.utils.distribution.plot import feature_distribution_traces, drift_score_bar_traces
 from deepchecks.utils.features import N_TOP_MESSAGE, calculate_feature_importance_or_none
 from deepchecks.utils.strings import format_percent
diff --git a/deepchecks/tabular/dataset.py b/deepchecks/tabular/dataset.py
--- a/deepchecks/tabular/dataset.py
+++ b/deepchecks/tabular/dataset.py
@@ -321,7 +321,7 @@ def from_numpy(
         Examples
         --------
         >>> import numpy
-        >>> from deepchecks import Dataset
+        >>> from deepchecks.tabular import Dataset
 
         >>> features = numpy.array([[0.25, 0.3, 0.3],
         ...                        [0.14, 0.75, 0.3],


Test patch:
diff --git a/deepchecks/vision/checks/distribution/train_test_label_drift.py b/deepchecks/vision/checks/distribution/train_test_label_drift.py
--- a/deepchecks/vision/checks/distribution/train_test_label_drift.py
+++ b/deepchecks/vision/checks/distribution/train_test_label_drift.py
@@ -14,7 +14,7 @@
 import pandas as pd
 from collections import OrderedDict, defaultdict
 
-from deepchecks import ConditionResult
+from deepchecks.core import ConditionResult
 from deepchecks.utils.distribution.drift import calc_drift_and_plot
 from deepchecks.core import DatasetKind, CheckResult
 from deepchecks.core.errors import DeepchecksNotSupportedError
diff --git a/tests/base/display_test.py b/tests/base/display_test.py
--- a/tests/base/display_test.py
+++ b/tests/base/display_test.py
@@ -20,7 +20,7 @@
 from ipywidgets import VBox
 
 from deepchecks.core.errors import DeepchecksValueError
-from deepchecks.checks import ColumnsInfo, DataDuplicates, MixedNulls
+from deepchecks.tabular.checks import ColumnsInfo, DataDuplicates, MixedNulls
 
 pio.renderers.default = "json"
Repository: deepchecks/deepchecks

Issue:
[FEAT] Exclude tests from the package distribution

Currently, the tests folder is shipped with the dist package. This shouldn't happen.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -158,7 +158,7 @@ def read_requirements() -> t.Dict[str,t.List[str]]:
     },
     
     # -- dependencies --------------------------------
-    packages=setuptools.find_packages(),
+    packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
     install_requires=main_requirements,
     extras_require=extra_requirements,
     dependency_links=dependency_links,


Test patch:
Repository: deepchecks/deepchecks

Issue:
[FEAT][CV] Add a "per-class" option to property drift & heatmap comparison
In this per class option, the drift would be shown per class for the top drifted classes. 


Discussion:
there's a bug in the per class heatmap, may increase time.
also might take a while anyways.

Resolving patch:
diff --git a/docs/source/examples/vision/checks/distribution/source/plot_image_property_check.py b/docs/source/examples/vision/checks/distribution/source/plot_image_property_check.py
--- a/docs/source/examples/vision/checks/distribution/source/plot_image_property_check.py
+++ b/docs/source/examples/vision/checks/distribution/source/plot_image_property_check.py
@@ -76,6 +76,17 @@
 
 check_result.value
 
+#%%
+# We can also pass the check a list of classes we wish to inspect, and the check will calculate the properties only
+# for images either belonging to the classes or containing annotations belonging to the classes. (We'll lower the
+# min_samples to 5 to tell the check to calculate drift despite having only a few images left after the class
+# filtration)
+
+check_result = ImagePropertyDrift(classes_to_display=['person', 'cat', 'cell phone', 'car'], min_samples=5
+                                  ).run(train_dataset, test_dataset)
+check_result
+
+
 #%%
 # Define a condition
 # ==================


Test patch: