Repository: deepchecks/deepchecks

Issue:
[FEAT][CV] Add method to run vision train_test_split checks on images in filesystem
**Is your feature request related to a problem? Please describe.**
Currently integration cost with deepchecks.vision is high (writing Data objects). For checks that don't need the model, it's possible to have a lower friction option to get checks results based on just images in the filesystem. 

**Describe the solution you'd like**
For classification - If images are organized in the most straightforward way (1 folder for train, 1 folder for test, in each images are inside folders with the class name) then a ImageFolder Dataset can be easily built and a default ClassificationData object used to get suite results for these images.
This can be run either as a python method that accepts two folder paths, or a python program that can be run from cmd with the two folder paths as inputs.




Discussion:

Resolving patch:
diff --git a/deepchecks/vision/datasets/classification/__init__.py b/deepchecks/vision/datasets/classification/__init__.py
--- a/deepchecks/vision/datasets/classification/__init__.py
+++ b/deepchecks/vision/datasets/classification/__init__.py
@@ -10,6 +10,6 @@
 #
 """Module for classification datasets and models."""
 from . import mnist
+from . import simple
 
-
-__all__ = ["mnist"]
+__all__ = ["mnist", "simple"]
diff --git a/deepchecks/vision/datasets/classification/simple.py b/deepchecks/vision/datasets/classification/simple.py
new file mode 100644
--- /dev/null
+++ b/deepchecks/vision/datasets/classification/simple.py
@@ -0,0 +1,211 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021-2022 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+"""Module contains implementation of the simple classification dataset."""
+import typing as t
+from pathlib import Path
+
+import cv2
+import PIL.Image as pilimage
+import torch
+import numpy as np
+from torch.utils.data import DataLoader
+from torchvision.datasets import VisionDataset
+from typing_extensions import Literal
+
+from deepchecks import vision
+
+
+__all__ = ['load_dataset', 'SimpleClassificationDataset']
+
+
+def load_dataset(
+    root: str,
+    train: bool = True,
+    batch_size: int = 32,
+    num_workers: int = 0,
+    shuffle: bool = True,
+    pin_memory: bool = True,
+    object_type: Literal['VisionData', 'DataLoader'] = 'DataLoader',
+    **kwargs
+) -> t.Union[DataLoader, vision.ClassificationData]:
+    """Load a simple classification dataset.
+
+    The function expects that the data within the root folder
+    to be structured in the following way:
+
+        - root/
+            - train/
+                - class1/
+                    image1.jpeg
+            - test/
+                - class1/
+                    image1.jpeg
+
+    Parameters
+    ----------
+    root : str
+        path to the data
+    train : bool
+        if `True` load the train dataset, otherwise load the test dataset
+    batch_size : int, default: 32
+        Batch size for the dataloader.
+    num_workers : int, default: 0
+        Number of workers for the dataloader.
+    shuffle : bool, default: True
+        Whether to shuffle the dataset.
+    pin_memory : bool, default: True
+        If ``True``, the data loader will copy Tensors
+        into CUDA pinned memory before returning them.
+    object_type : Literal['Dataset', 'DataLoader'], default: 'DataLoader'
+        type of the return value. If 'Dataset', :obj:`deepchecks.vision.VisionDataset`
+        will be returned, otherwise :obj:`torch.utils.data.DataLoader`
+
+    Returns
+    -------
+    Union[DataLoader, VisionDataset]
+
+        A DataLoader or VisionDataset instance representing the dataset
+    """
+
+    def batch_collate(batch):
+        imgs, labels = zip(*batch)
+        return list(imgs), list(labels)
+
+    dataloader = DataLoader(
+        dataset=SimpleClassificationDataset(root=root, train=train, **kwargs),
+        batch_size=batch_size,
+        shuffle=shuffle,
+        num_workers=num_workers,
+        collate_fn=batch_collate,
+        pin_memory=pin_memory,
+        generator=torch.Generator()
+    )
+
+    if object_type == 'DataLoader':
+        return dataloader
+    elif object_type == 'VisionData':
+        return SimpleClassificationData(data_loader=dataloader)
+    else:
+        raise TypeError(f'Unknown value of object_type - {object_type}')
+
+
+class SimpleClassificationDataset(VisionDataset):
+    """Simple VisionDataset type for the classification tasks.
+
+    The current class expects that data within the root folder
+    will be structured in a next way:
+
+        - root/
+            - train/
+                - class1/
+                    image1.jpeg
+            - test/
+                - class1/
+                    image1.jpeg
+
+    Otherwise exception will be raised.
+
+    Parameters
+    ----------
+    root : str
+        Path to the root directory of the dataset.
+    train : bool
+        if `True` train dataset, otherwise test dataset
+    transforms : Callable, optional
+        A function/transform that takes in an PIL image and returns a transformed version.
+        E.g, transforms.RandomCrop
+    transform : Callable, optional
+        A function/transforms that takes in an image and a label and returns the
+        transformed versions of both.
+        E.g, ``transforms.Rotate``
+    target_transform : Callable, optional
+        A function/transform that takes in the target and transforms it.
+    image_extension : str, default 'JPEG'
+        images format
+    """
+
+    def __init__(
+        self,
+        root: str,
+        train: bool = True,
+        transforms: t.Optional[t.Callable] = None,
+        transform: t.Optional[t.Callable] = None,
+        target_transform: t.Optional[t.Callable] = None,
+        image_extension: str = 'JPEG'
+    ) -> None:
+        self.root_path = Path(root).absolute()
+
+        if not (self.root_path.exists() and self.root_path.is_dir()):
+            raise ValueError(f'{self.root_path} - path does not exist or is not a folder')
+
+        super().__init__(str(self.root_path), transforms, transform, target_transform)
+        self.image_extension = image_extension.lower()
+
+        if train is True:
+            self.images = sorted(self.root_path.glob(f'train/*/*.{self.image_extension}'))
+        else:
+            self.images = sorted(self.root_path.glob(f'test/*/*.{self.image_extension}'))
+
+        if len(self.images) == 0:
+            raise ValueError(f'{self.root_path} - is empty or has incorrect structure')
+
+        # class label -> class index
+        self.classes_map = t.cast(t.Dict[str, int], {
+            img.parent.name: index
+            for index, img in enumerate(self.images)
+        })
+        # class index -> class  label
+        self.reverse_classes_map = t.cast(t.Dict[int, str], {
+            v: k
+            for k, v in self.classes_map.items()
+        })
+
+    def __getitem__(self, index: int) -> t.Tuple[np.ndarray, int]:
+        """Get the image and label at the given index."""
+        image_file = self.images[index]
+        image = cv2.imread(str(image_file))
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        target = self.classes_map[image_file.parent.name]
+
+        if self.transforms is not None:
+            transformed = self.transforms(image=image, target=target)
+            image, target = transformed['image'], transformed['target']
+        else:
+            if self.transform is not None:
+                image = self.transform(image=image)['image']
+            if self.target_transform is not None:
+                target = self.target_transform(target)
+
+        return image, target
+
+    def __len__(self) -> int:
+        """Return the number of images in the dataset."""
+        return len(self.images)
+
+
+class SimpleClassificationData(vision.ClassificationData):
+    """Simple ClassificationData type, matches the data returned by SimpleClassificationDataset getitem."""
+
+    def batch_to_images(
+        self,
+        batch: t.Tuple[t.Sequence[np.ndarray], t.Sequence[int]]
+    ) -> t.Sequence[np.ndarray]:
+        """Extract the images from a batch of data."""
+        images, _ = batch
+        return images
+
+    def batch_to_labels(
+        self,
+        batch: t.Tuple[t.Sequence[pilimage.Image], t.Sequence[int]]
+    ) -> torch.Tensor:
+        """Extract the labels from a batch of data."""
+        _, labels = batch
+        return torch.Tensor(list(labels))


Test patch:
diff --git a/tests/vision/datasets/test_simple.py b/tests/vision/datasets/test_simple.py
new file mode 100644
--- /dev/null
+++ b/tests/vision/datasets/test_simple.py
@@ -0,0 +1,115 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021-2022 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+import pathlib
+from shutil import rmtree
+
+import numpy as np
+import pytest
+import PIL.Image as pilimage
+from torch.utils.data import DataLoader
+from hamcrest import assert_that, instance_of, has_length, all_of, has_property, equal_to, calling, raises
+
+from deepchecks.utils.strings import get_random_string
+from deepchecks.vision.datasets.classification import simple
+
+
+PARENT_FOLDER = pathlib.Path(__file__).absolute().parent
+
+
+@pytest.fixture
+def correct_images_folder():
+    image = pilimage.fromarray(np.zeros((10, 10, 3), dtype=np.uint8))
+
+    images_folder = PARENT_FOLDER / get_random_string()
+    images_folder.mkdir()
+    (images_folder / "train").mkdir()
+    (images_folder / "train" / "class1").mkdir()
+    (images_folder / "test").mkdir()
+    (images_folder / "test" / "class1").mkdir()
+
+    image.save(str(images_folder / "train" / "class1" / "image.png"))
+    image.save(str(images_folder / "test" / "class1" / "image.png"))
+
+    yield images_folder
+
+    rmtree(str(images_folder))
+
+
+@pytest.fixture
+def incorrect_images_folder():
+    image = pilimage.fromarray(np.zeros((10, 10, 3), dtype=np.uint8))
+
+    images_folder = PARENT_FOLDER / get_random_string()
+    images_folder.mkdir()
+    (images_folder / "train").mkdir()
+    (images_folder / "test").mkdir()
+
+    image.save(str(images_folder / "train" / "image.png"))
+    image.save(str(images_folder / "test" / "image.png"))
+
+    yield images_folder
+
+    rmtree(str(images_folder))
+
+
+def test_load_simple_classification_dataset(correct_images_folder):
+    dataloader = simple.load_dataset(
+        root=str(correct_images_folder),
+        object_type="DataLoader",
+        image_extension="PNG"
+    )
+
+    assert_that(dataloader, all_of(
+        instance_of(DataLoader),
+        has_length(1),
+        has_property("dataset", instance_of(simple.SimpleClassificationDataset))
+    ))
+
+    img, label = dataloader.dataset[0]
+
+    assert_that(img, all_of(
+        instance_of(np.ndarray),
+        has_property("shape", equal_to((10, 10, 3)))
+    ))
+
+    assert_that(label, instance_of(int))
+    assert_that(dataloader.dataset.reverse_classes_map[label] == "class1")
+
+
+def test_load_simple_classification_vision_data(correct_images_folder):
+    vision_data = simple.load_dataset(
+        root=str(correct_images_folder),
+        object_type="VisionData",
+        image_extension="PNG"
+    )
+
+    batches = list(vision_data)
+    assert_that(len(batches) == 1)
+
+    images = vision_data.batch_to_images(batches[0])
+    labels = vision_data.batch_to_labels(batches[0])
+    assert_that(len(images) == 1 and len(labels) == 1)
+
+    assert_that(images[0], all_of(
+        instance_of(np.ndarray),
+        has_property("shape", equal_to((10, 10, 3)))
+    ))
+    assert_that(vision_data._data_loader.dataset.reverse_classes_map[labels[0].item()] == "class1")
+
+
+def test_load_simple_classification_dataset_from_broken_folder(incorrect_images_folder):
+    assert_that(
+        calling(simple.load_dataset).with_args(
+            root=str(incorrect_images_folder),
+            object_type="DataLoader",
+            image_extension="PNG"),
+        raises(ValueError)
+    )
\ No newline at end of file