Repository: Project-MONAI/MONAI

Issue:
test_wsireader full tests errors
**Describe the bug**
```
======================================================================
ERROR: test_read_rgba_0 (tests.test_wsireader.TestTiffFile)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/parameterized/parameterized.py", line 533, in standalone_func
    return func(*(a + p.args), **p.kwargs)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/tests/test_wsireader.py", line 158, in test_read_rgba
    image[mode], _ = reader.get_data(img_obj)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/image_reader.py", line 775, in get_data
    region = self._extract_region(img, location=location, size=size, level=level, dtype=dtype)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/image_reader.py", line 850, in _extract_region
    region = region[location[0] : location[0] + size[0], location[1] : location[1] + size[1]]
TypeError: slice indices must be integers or None or have an __index__ method

======================================================================
ERROR: test_read_rgba_1 (tests.test_wsireader.TestTiffFile)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/parameterized/parameterized.py", line 533, in standalone_func
    return func(*(a + p.args), **p.kwargs)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/tests/test_wsireader.py", line 158, in test_read_rgba
    image[mode], _ = reader.get_data(img_obj)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/image_reader.py", line 775, in get_data
    region = self._extract_region(img, location=location, size=size, level=level, dtype=dtype)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/image_reader.py", line 850, in _extract_region
    region = region[location[0] : location[0] + size[0], location[1] : location[1] + size[1]]
TypeError: slice indices must be integers or None or have an __index__ method

======================================================================
ERROR: test_read_whole_image_0__home_jenkins_agent_workspace_MONAI_postmerge_monai_github_tests_testing_data_temp_1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe_tiff (tests.test_wsireader.TestTiffFile)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/parameterized/parameterized.py", line 533, in standalone_func
    return func(*(a + p.args), **p.kwargs)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/tests/test_wsireader.py", line 117, in test_read_whole_image
    img = reader.get_data(img_obj)[0]
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/image_reader.py", line 775, in get_data
    region = self._extract_region(img, location=location, size=size, level=level, dtype=dtype)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/image_reader.py", line 850, in _extract_region
    region = region[location[0] : location[0] + size[0], location[1] : location[1] + size[1]]
TypeError: slice indices must be integers or None or have an __index__ method

======================================================================
ERROR: test_with_dataloader_0__home_jenkins_agent_workspace_MONAI_postmerge_monai_github_tests_testing_data_temp_1sGTKZlJBIz53pfqTxoTqiIQzIoEzHLAe_tiff (tests.test_wsireader.TestTiffFile)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/transforms/transform.py", line 82, in apply_transform
    return _apply_transform(transform, data, unpack_items)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/transforms/transform.py", line 53, in _apply_transform
    return transform(parameters)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/transforms/io/dictionary.py", line 121, in __call__
    data = self._loader(d[key], reader)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/transforms/io/array.py", line 214, in __call__
    img_array, meta_data = reader.get_data(img)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/image_reader.py", line 775, in get_data
    region = self._extract_region(img, location=location, size=size, level=level, dtype=dtype)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/image_reader.py", line 850, in _extract_region
    region = region[location[0] : location[0] + size[0], location[1] : location[1] + size[1]]
TypeError: slice indices must be integers or None or have an __index__ method

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/transforms/transform.py", line 82, in apply_transform
    return _apply_transform(transform, data, unpack_items)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/transforms/transform.py", line 53, in _apply_transform
    return transform(parameters)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/transforms/compose.py", line 160, in __call__
    input_ = apply_transform(_transform, input_, self.map_items, self.unpack_items)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/transforms/transform.py", line 106, in apply_transform
    raise RuntimeError(f"applying transform {transform}") from e
RuntimeError: applying transform <monai.transforms.io.dictionary.LoadImaged object at 0x7f2571aca940>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/opt/conda/lib/python3.6/site-packages/parameterized/parameterized.py", line 533, in standalone_func
    return func(*(a + p.args), **p.kwargs)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/tests/test_wsireader.py", line 173, in test_with_dataloader
    data: dict = first(data_loader)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/utils/misc.py", line 73, in first
    for i in iterable:
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 521, in __next__
    data = self._next_data()
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 561, in _next_data
    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/opt/conda/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py", line 49, in <listcomp>
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/dataset.py", line 97, in __getitem__
    return self._transform(index)
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/data/dataset.py", line 83, in _transform
    return apply_transform(self.transform, data_i) if self.transform is not None else data_i
  File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/transforms/transform.py", line 106, in apply_transform
    raise RuntimeError(f"applying transform {transform}") from e
RuntimeError: applying transform <monai.transforms.compose.Compose object at 0x7f23c856a7f0>

----------------------------------------------------------------------
```

env:
```
MONAI version: 0+untagged.251.g5391f7c
Numpy version: 1.18.1
Pytorch version: 1.10.0+cu102
MONAI flags: HAS_EXT = True, USE_COMPILED = True
MONAI rev id: 5391f7cded378f30b0ce0758b19adcf9daf988c8

Optional dependencies:
Pytorch Ignite version: 0.4.6
Nibabel version: 3.2.1
scikit-image version: 0.15.0
Pillow version: 8.4.0
Tensorboard version: 2.1.0
gdown version: 4.2.0
TorchVision version: 0.11.1+cu102
tqdm version: 4.62.3
lmdb version: 0.98
psutil version: 5.6.7
pandas version: 0.24.2
einops version: 0.3.2
transformers version: 4.12.5
mlflow version: 1.22.0
```


Discussion:
caused by https://github.com/Project-MONAI/MONAI/pull/3417, could you please look into this? @drbeh
@wyli, I'll take a look at this but do you know how should I reproduce it?

Also do you know why we have not been able to see this in all the checks that we have done before merging?!
> Also do you know why we have not been able to see this in all the checks that we have done before merging?!

yes, we only run basic tests before merging so that we handle PRs quickly.. after merging the PRs we have more tests against various environments...
the test script is roughly with these commands:
```bash
python -m pip uninstall -y torch torchvision
python -m pip install torch==1.10.0 torchvision==0.11.1
python -m pip install -r requirements-dev.txt
BUILD_MONAI=1 ./runtests.sh --unittests
```
on a v100 GPU, within a docker `nvcr.io/nvidia/pytorch:20.03-py3`

please let me know if these couldn't replicate it

Resolving patch:
diff --git a/monai/data/image_reader.py b/monai/data/image_reader.py
--- a/monai/data/image_reader.py
+++ b/monai/data/image_reader.py
@@ -768,10 +768,8 @@ def get_data(
         # Verify inputs
         if level is None:
             level = self._check_level(img, level)
-        if size is None:
-            size = self._get_image_size(img, size, level, location)
 
-        # Extract patch (or the whole image)
+        # Extract a region or the entire image
         region = self._extract_region(img, location=location, size=size, level=level, dtype=dtype)
 
         # Add necessary metadata
@@ -810,49 +808,56 @@ def _check_level(self, img, level):
         return level
 
     def _get_image_size(self, img, size, level, location):
+        """
+        Calculate the maximum region size for the given level and starting location (if size is None).
+        Note that region size in OpenSlide and cuCIM are WxH (but the final image output would be HxW)
+        """
+        if size is not None:
+            return size[::-1]
+
         max_size = []
         downsampling_factor = []
         if self.backend == "openslide":
             downsampling_factor = img.level_downsamples[level]
-            max_size = img.level_dimensions[level][::-1]
+            max_size = img.level_dimensions[level]
         elif self.backend == "cucim":
             downsampling_factor = img.resolutions["level_downsamples"][level]
-            max_size = img.resolutions["level_dimensions"][level][::-1]
-        elif self.backend == "tifffile":
-            level0_size = img.pages[0].shape[:2]
-            max_size = img.pages[level].shape[:2]
-            downsampling_factor = np.mean([level0_size[i] / max_size[i] for i in range(len(max_size))])
+            max_size = img.resolutions["level_dimensions"][level]
 
-        # subtract the top left corner of the patch from maximum size
-        level_location = [round(location[i] / downsampling_factor) for i in range(len(location))]
-        size = [max_size[i] - level_location[i] for i in range(len(max_size))]
+        # subtract the top left corner of the patch (at given level) from maximum size
+        location_at_level = (round(location[1] / downsampling_factor), round(location[0] / downsampling_factor))
+        size = [max_size[i] - location_at_level[i] for i in range(len(max_size))]
 
         return size
 
     def _extract_region(
         self,
         img_obj,
-        size: Tuple[int, int],
+        size: Optional[Tuple[int, int]],
         location: Tuple[int, int] = (0, 0),
         level: int = 0,
         dtype: DtypeLike = np.uint8,
     ):
         if self.backend == "tifffile":
-            # with img_obj:
-            region = img_obj.asarray(level=level)
-            if level != 0:
-                level0_size = img_obj.pages[0].shape[:2]
-                max_size = img_obj.pages[level].shape[:2]
-                location = (
-                    int(location[0] / level0_size[0] * max_size[0]),
-                    int(location[1] / level0_size[1] * max_size[1]),
+            # Read the entire image
+            if size is not None:
+                raise ValueError(
+                    f"TiffFile backend reads the entire image only, so size '{size}'' should not be provided!",
+                    "For more flexibility or extracting regions, please use cuCIM or OpenSlide backend.",
+                )
+            if location != (0, 0):
+                raise ValueError(
+                    f"TiffFile backend reads the entire image only, so location '{location}' should not be provided!",
+                    "For more flexibility and extracting regions, please use cuCIM or OpenSlide backend.",
                 )
-            region = region[location[0] : location[0] + size[0], location[1] : location[1] + size[1]]
+            region = img_obj.asarray(level=level)
         else:
-            # reverse the order of dimensions for size and location to become WxH
-            location = location[::-1]
-            size = size[::-1]
-            region = img_obj.read_region(location=location, size=size, level=level)
+            # Get region size to be extracted
+            region_size = self._get_image_size(img_obj, size, level, location)
+            # reverse the order of location's dimensions to become WxH (for cuCIM and OpenSlide)
+            region_location = location[::-1]
+            # Extract a region (or the entire image)
+            region = img_obj.read_region(location=region_location, size=region_size, level=level)
 
         region = self.convert_to_rgb_array(region, dtype)
         return region


Test patch:
diff --git a/tests/test_wsireader.py b/tests/test_wsireader.py
--- a/tests/test_wsireader.py
+++ b/tests/test_wsireader.py
@@ -119,26 +119,31 @@ def test_read_whole_image(self, file_path, level, expected_shape):
 
         @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_5])
         def test_read_region(self, file_path, patch_info, expected_img):
-            # Due to CPU memory limitation ignore tifffile at level 0.
-            if self.backend == "tifffile" and patch_info["level"] == 0:
-                return
             reader = WSIReader(self.backend)
-            # Read twice to check multiple calls
             with reader.read(file_path) as img_obj:
-                img = reader.get_data(img_obj, **patch_info)[0]
-                img2 = reader.get_data(img_obj, **patch_info)[0]
-            self.assertTupleEqual(img.shape, img2.shape)
-            self.assertIsNone(assert_array_equal(img, img2))
-            self.assertTupleEqual(img.shape, expected_img.shape)
-            self.assertIsNone(assert_array_equal(img, expected_img))
+                if self.backend == "tifffile":
+                    with self.assertRaises(ValueError):
+                        reader.get_data(img_obj, **patch_info)[0]
+                else:
+                    # Read twice to check multiple calls
+                    img = reader.get_data(img_obj, **patch_info)[0]
+                    img2 = reader.get_data(img_obj, **patch_info)[0]
+                    self.assertTupleEqual(img.shape, img2.shape)
+                    self.assertIsNone(assert_array_equal(img, img2))
+                    self.assertTupleEqual(img.shape, expected_img.shape)
+                    self.assertIsNone(assert_array_equal(img, expected_img))
 
         @parameterized.expand([TEST_CASE_3, TEST_CASE_4])
         def test_read_patches(self, file_path, patch_info, expected_img):
             reader = WSIReader(self.backend)
             with reader.read(file_path) as img_obj:
-                img = reader.get_data(img_obj, **patch_info)[0]
-            self.assertTupleEqual(img.shape, expected_img.shape)
-            self.assertIsNone(assert_array_equal(img, expected_img))
+                if self.backend == "tifffile":
+                    with self.assertRaises(ValueError):
+                        reader.get_data(img_obj, **patch_info)[0]
+                else:
+                    img = reader.get_data(img_obj, **patch_info)[0]
+                    self.assertTupleEqual(img.shape, expected_img.shape)
+                    self.assertIsNone(assert_array_equal(img, expected_img))
 
         @parameterized.expand([TEST_CASE_RGB_0, TEST_CASE_RGB_1])
         @skipUnless(has_tiff, "Requires tifffile.")