Repository: comic/grand-challenge.org

Issue:
Upload of Phillips tiff files fails 
When uploading tiff files from a Philps scanner, there are a number of errors that occur while uploading. The files are also not shown properly by MeVisLab


Discussion:
Will adding OpenSlide with their python bindings fix this? 
Yes, jusr have to get some info about the image in a different way. 

Resolving patch:
diff --git a/app/grandchallenge/cases/image_builders/tiff.py b/app/grandchallenge/cases/image_builders/tiff.py
--- a/app/grandchallenge/cases/image_builders/tiff.py
+++ b/app/grandchallenge/cases/image_builders/tiff.py
@@ -1,7 +1,8 @@
+from dataclasses import dataclass
 from pathlib import Path
 from tempfile import TemporaryFile
-from typing import NamedTuple, Union
-from uuid import uuid4
+from typing import Optional
+from uuid import UUID, uuid4
 
 import pyvips
 import tifffile
@@ -13,22 +14,33 @@
 from grandchallenge.cases.models import FolderUpload, Image, ImageFile
 
 
-class GrandChallengeTiffFileTags(NamedTuple):
-    image_width: int
-    image_height: int
-    resolution_levels: int
-    color_space: str
-    voxel_width_mm: float
-    voxel_height_mm: float
-    voxel_depth_mm: Union[float, None]
-
-
-class GrandChallengeTiffFile(NamedTuple):
+@dataclass
+class GrandChallengeTiffFile:
     path: Path
-    tags: GrandChallengeTiffFileTags
-
-
-def load_tiff_file(*, path: Path) -> GrandChallengeTiffFile:
+    image_width: int = 0
+    image_height: int = 0
+    resolution_levels: int = 0
+    color_space: str = ""
+    voxel_width_mm: float = 0
+    voxel_height_mm: float = 0
+    voxel_depth_mm: Optional[float] = None
+
+    def validate(self):
+        if not self.image_width:
+            raise ValidationError("ImageWidth could not be determined")
+        if not self.image_height:
+            raise ValidationError("ImageHeigth could not be determined")
+        if not self.resolution_levels:
+            raise ValidationError("Resolution levels not valid")
+        if not self.color_space:
+            raise ValidationError("ColorSpace not valid")
+        if not self.voxel_width_mm:
+            raise ValidationError("Voxel width could not be determined")
+        if not self.voxel_height_mm:
+            raise ValidationError("Voxel heigth could not be determined")
+
+
+def load_tiff_file(path: Path) -> GrandChallengeTiffFile:
     """
     Loads and validates a file using tifffile
     :param path: The path to the potential tiff file
@@ -36,100 +48,88 @@ def load_tiff_file(*, path: Path) -> GrandChallengeTiffFile:
     """
     try:
         file = tifffile.TiffFile(str(path.absolute()))
+        gc_file = GrandChallengeTiffFile(path)
     except ValueError:
         raise ValidationError("Image isn't a TIFF file")
 
-    tags = _validate_tifffile(pages=file.pages)
+    return _validate_tiff_file(gc_file=gc_file, pages=file.pages)
 
-    return GrandChallengeTiffFile(path=path, tags=tags)
 
+def _get_tag_value(tags, tag, required=True):
+    try:
+        return tags[tag].value
+    except KeyError:
+        if required:
+            raise ValidationError(f"Tiff file is missing required tag {tag}")
 
-def _validate_tifffile(  # noqa: C901
-    *, pages: tifffile.tifffile.TiffPages
-) -> GrandChallengeTiffFileTags:
-    """
-    Validates a tiff file loaded with tifffile for use in grand challenge
-    :param pages: The pages and tags from tiffile
-    :return: The extracted tags that are needed by the rest of the framework
+
+def _get_voxel_spacing_mm(tags, tag):
     """
+    Calculate the voxel spacing in mm.
 
-    def get_tag_value(tags, tag, required=True):
-        try:
-            return tags[tag].value
-        except KeyError:
-            if required:
-                raise ValidationError(
-                    f"Tiff file is missing required tag {tag}"
-                )
+    Use the set of tags from the tiff image to calculate the spacing for a
+    particular dimension. Supports INCH and CENTIMETER resolution units.
 
-    def get_voxel_spacing_mm(tags, tag):
-        """
-        Calculate the voxel spacing in mm.
+    Parameters
+    ----------
+    tags
+        The collection of tags from the tif file.
+    tag
+        The tag that contains the resolution of the tif file along the
+        dimension of interest.
 
-        Use the set of tags from the tiff image to calculate the spacing for a
-        particular dimension. Supports INCH and CENTIMETER resolution units.
+    Raises
+    ------
+    ValidationError
+        Raised if an unrecognised resolution unit is used.
 
-        Parameters
-        ----------
-        tags
-            The collection of tags from the tif file.
-        tag
-            The tag that contains the resolution of the tif file along the
-            dimension of interest.
 
-        Raises
-        ------
-        ValidationError
-            Raised if an unrecognised resolution unit is used.
+    Returns
+    -------
+        The voxel spacing in mm.
 
+    """
+    try:
+        # Resolution is a tuple of the number of pixels and the length of the
+        # image in cm or inches, depending on resolution unit
+        resolution_unit = str(_get_tag_value(tags, "ResolutionUnit"))
+        resolution = _get_tag_value(tags, tag)
+        if resolution_unit == "RESUNIT.INCH":
+            return 25.4 / (resolution[0] / resolution[1])
+        elif resolution_unit == "RESUNIT.CENTIMETER":
+            return 10 / (resolution[0] / resolution[1])
+        raise ValidationError(
+            f"Invalid resolution unit {resolution_unit}" f" in tiff file"
+        )
+    except ZeroDivisionError:
+        raise ValidationError(f"Invalid resolution in tiff file")
 
-        Returns
-        -------
-            The voxel spacing in mm.
 
-        """
-        try:
-            # Resolution is a tuple of the number of pixels and the length of the
-            # image in cm or inches, depending on resolution unit
-            resolution_unit = str(get_tag_value(tags, "ResolutionUnit"))
-            resolution = get_tag_value(tags, tag)
-            if resolution_unit == "RESUNIT.INCH":
-                return 25.4 / (resolution[0] / resolution[1])
-            elif resolution_unit == "RESUNIT.CENTIMETER":
-                return 10 / (resolution[0] / resolution[1])
-            raise ValidationError(
-                f"Invalid resolution unit {resolution_unit}" f" in tiff file"
-            )
-        except ZeroDivisionError:
-            raise ValidationError(f"Invalid resolution in tiff file")
+def _validate_tiff_file(
+    *, gc_file: GrandChallengeTiffFile, pages: tifffile.tifffile.TiffPages
+) -> GrandChallengeTiffFile:
+    """
+    Validates a tiff file loaded with tifffile for use in grand challenge
+    :param pages: The pages and tags from tiffile
+    :return: The extracted tags that are needed by the rest of the framework
+    """
 
     required_tile_tags = ("TileOffsets", "TileByteCounts")
 
-    forbidden_description_tags = ("dicom", "xml")
-
     tags = pages[0].tags
 
-    # Checks if the image description exists,
-    # if so, ensure there's no DICOM or XML data
-    image_description = str(
-        get_tag_value(tags, "ImageDescription", False)
-    ).lower()
-    for forbidden in forbidden_description_tags:
-        if forbidden in image_description:
-            raise ValidationError("Image contains unauthorized information")
-
     # Fails if the image doesn't have all required tile tags
     for tag in required_tile_tags:
-        get_tag_value(tags, tag, True)
+        _get_tag_value(tags, tag, True)
 
     # Fails if the image only has a single resolution page
-    resolution_levels = len(pages)
-    if resolution_levels == 1:
+    gc_file.resolution_levels = len(pages)
+    if gc_file.resolution_levels == 1:
         raise ValidationError("Image only has a single resolution level")
 
     # Fails if the image doesn't have the chunky format
     if (
-        str(get_tag_value(tags, "PlanarConfiguration"))
+        str(_get_tag_value(tags, "PlanarConfiguration"))
         != "PLANARCONFIG.CONTIG"
     ):
         raise ValidationError(
@@ -137,33 +137,25 @@ def get_voxel_spacing_mm(tags, tag):
         )
 
     # Fails if the color space isn't supported
-    color_space = get_color_space(
-        str(get_tag_value(tags, "PhotometricInterpretation"))
+    gc_file.color_space = _get_color_space(
+        color_space_string=str(
+            _get_tag_value(tags, "PhotometricInterpretation")
+        )
     )
+    gc_file.image_width = _get_tag_value(tags, "ImageWidth")
+    gc_file.image_height = _get_tag_value(tags, "ImageLength")
 
-    # Fails if the amount of bytes per sample doesn't correspond to the
-    # colour space
-    tif_color_channels = get_tag_value(tags, "SamplesPerPixel")
-    if Image.COLOR_SPACE_COMPONENTS[color_space] != tif_color_channels:
-        raise ValidationError("Image contains invalid amount of channels.")
-
-    voxel_width_mm = get_voxel_spacing_mm(tags, "XResolution")
-    voxel_height_mm = get_voxel_spacing_mm(tags, "YResolution")
-    image_width = get_tag_value(tags, "ImageWidth")
-    image_height = get_tag_value(tags, "ImageLength")
-
-    return GrandChallengeTiffFileTags(
-        image_width=image_width,
-        image_height=image_height,
-        color_space=color_space,
-        resolution_levels=resolution_levels,
-        voxel_width_mm=voxel_width_mm,
-        voxel_height_mm=voxel_height_mm,
-        voxel_depth_mm=None,
-    )
+    #  some formats like the Philips tiff don't have the spacing in their tags,
+    #  we retrieve them later with OpenSlide
+    if "XResolution" in tags:
+        gc_file.voxel_width_mm = _get_voxel_spacing_mm(tags, "XResolution")
+        gc_file.voxel_height_mm = _get_voxel_spacing_mm(tags, "YResolution")
+    gc_file.voxel_depth_mm = None
 
+    return gc_file
 
-def get_color_space(color_space_string) -> Image.COLOR_SPACES:
+
+def _get_color_space(*, color_space_string) -> Image.COLOR_SPACES:
     color_space_string = color_space_string.split(".")[1].upper()
 
     if color_space_string == "MINISBLACK":
@@ -177,6 +169,28 @@ def get_color_space(color_space_string) -> Image.COLOR_SPACES:
     return color_space
 
 
+def _create_image_file(*, path: str, image: Image):
+    temp_file = TemporaryFile()
+    with open(path, "rb") as open_file:
+        buffer = True
+        while buffer:
+            buffer = open_file.read(1024)
+            temp_file.write(buffer)
+
+    if path.lower().endswith("dzi"):
+        return ImageFile(
+            image=image,
+            image_type=ImageFile.IMAGE_TYPE_DZI,
+            file=File(temp_file, name=f"{image.pk}.dzi"),
+        )
+    else:
+        return ImageFile(
+            image=image,
+            image_type=ImageFile.IMAGE_TYPE_TIFF,
+            file=File(temp_file, name=f"{image.pk}.tif"),
+        )
+
+
 def image_builder_tiff(path: Path, session_id=None) -> ImageBuilderResult:
     new_images = []
     new_image_files = []
@@ -188,47 +202,32 @@ def image_builder_tiff(path: Path, session_id=None) -> ImageBuilderResult:
         pk = uuid4()
         dzi_output = None
         try:
-            tiff_file = load_tiff_file(path=file_path)
+            gc_file = load_tiff_file(file_path)
         except ValidationError as e:
             invalid_file_errors[file_path.name] = e.message  # noqa: B306
             continue
 
         try:
-            dzi_output = create_dzi_images(tiff_file=tiff_file, pk=pk)
+            dzi_output, gc_file = _create_dzi_images(gc_file=gc_file, pk=pk)
         except ValidationError as e:
             invalid_file_errors[file_path.name] = e.message  # noqa: B306
 
-        image = create_tiff_image_entry(tiff_file=tiff_file, pk=pk)
+        # make sure all requirements are met
+        try:
+            gc_file.validate()
+        except ValidationError as e:
+            invalid_file_errors[file_path.name] = e.message  # noqa: B306
+            continue
 
-        temp_file = TemporaryFile()
-        with open(tiff_file.path.absolute(), "rb") as open_file:
-            buffer = True
-            while buffer:
-                buffer = open_file.read(1024)
-                temp_file.write(buffer)
+        image = _create_tiff_image_entry(tiff_file=gc_file, pk=pk)
 
         new_image_files.append(
-            ImageFile(
-                image=image,
-                image_type=ImageFile.IMAGE_TYPE_TIFF,
-                file=File(temp_file, name=f"{image.pk}.tif"),
-            )
+            _create_image_file(path=str(gc_file.path.absolute()), image=image)
         )
 
         if dzi_output:
-            temp_dzi_file = TemporaryFile()
-            with open(dzi_output + ".dzi", "rb") as open_file:
-                buffer = True
-                while buffer:
-                    buffer = open_file.read(1024)
-                    temp_dzi_file.write(buffer)
-
             new_image_files.append(
-                ImageFile(
-                    image=image,
-                    image_type=ImageFile.IMAGE_TYPE_DZI,
-                    file=File(temp_dzi_file, name=f"{image.pk}.dzi"),
-                )
+                _create_image_file(path=dzi_output + ".dzi", image=image)
             )
 
             dzi_folder_upload = FolderUpload(
@@ -237,7 +236,7 @@ def image_builder_tiff(path: Path, session_id=None) -> ImageBuilderResult:
             new_folder_upload.append(dzi_folder_upload)
 
         new_images.append(image)
-        consumed_files.add(tiff_file.path.name)
+        consumed_files.add(gc_file.path.name)
 
     return ImageBuilderResult(
         consumed_files=consumed_files,
@@ -248,32 +247,43 @@ def image_builder_tiff(path: Path, session_id=None) -> ImageBuilderResult:
     )
 
 
-def create_tiff_image_entry(*, tiff_file: GrandChallengeTiffFile, pk) -> Image:
+def _create_tiff_image_entry(
+    *, tiff_file: GrandChallengeTiffFile, pk: UUID
+) -> Image:
     # Builds a new Image model item
     return Image(
         pk=pk,
         name=tiff_file.path.name,
-        width=tiff_file.tags.image_width,
-        height=tiff_file.tags.image_height,
+        width=tiff_file.image_width,
+        height=tiff_file.image_height,
         depth=1,
-        resolution_levels=tiff_file.tags.resolution_levels,
-        color_space=tiff_file.tags.color_space,
+        resolution_levels=tiff_file.resolution_levels,
+        color_space=tiff_file.color_space,
         eye_choice=Image.EYE_UNKNOWN,
         stereoscopic_choice=Image.STEREOSCOPIC_UNKNOWN,
         field_of_view=Image.FOV_UNKNOWN,
-        voxel_width_mm=tiff_file.tags.voxel_width_mm,
-        voxel_height_mm=tiff_file.tags.voxel_height_mm,
-        voxel_depth_mm=tiff_file.tags.voxel_depth_mm,
+        voxel_width_mm=tiff_file.voxel_width_mm,
+        voxel_height_mm=tiff_file.voxel_height_mm,
+        voxel_depth_mm=tiff_file.voxel_depth_mm,
     )
 
 
-def create_dzi_images(*, tiff_file: GrandChallengeTiffFile, pk) -> str:
+def _create_dzi_images(
+    *, gc_file: GrandChallengeTiffFile, pk: UUID
+) -> (str, GrandChallengeTiffFile):
     # Creates a dzi file(out.dzi) and corresponding tiles in folder {pk}_files
-    dzi_output = str(tiff_file.path.parent / str(pk))
+    dzi_output = str(gc_file.path.parent / str(pk))
     try:
         image = pyvips.Image.new_from_file(
-            str(tiff_file.path.absolute()), access="sequential"
+            str(gc_file.path.absolute()), access="sequential"
         )
+        if not gc_file.voxel_width_mm:
+            gc_file.voxel_width_mm = (
+                float(image.get("openslide.mpp-x")) / 1000,
+            )
+            gc_file.voxel_height_mm = (
+                float(image.get("openslide.mpp-y")) / 1000,
+            )
 
         pyvips.Image.dzsave(
             image, dzi_output, tile_size=settings.DZI_TILE_SIZE
@@ -281,4 +291,4 @@ def create_dzi_images(*, tiff_file: GrandChallengeTiffFile, pk) -> str:
     except Exception as e:
         raise ValidationError("Image can't be converted to dzi: " + str(e))
 
-    return dzi_output
+    return dzi_output, gc_file


Test patch:
diff --git a/app/tests/cases_tests/test_tiff.py b/app/tests/cases_tests/test_tiff.py
--- a/app/tests/cases_tests/test_tiff.py
+++ b/app/tests/cases_tests/test_tiff.py
@@ -9,9 +9,9 @@
 from pytest import approx
 
 from grandchallenge.cases.image_builders.tiff import (
-    create_dzi_images,
-    create_tiff_image_entry,
-    get_color_space,
+    _create_dzi_images,
+    _create_tiff_image_entry,
+    _get_color_space,
     image_builder_tiff,
     load_tiff_file,
 )
@@ -35,7 +35,7 @@ def test_get_color_space(color_space_string, expected):
     color_space = None
 
     try:
-        color_space = get_color_space(color_space_string)
+        color_space = _get_color_space(color_space_string=color_space_string)
     except ValidationError:
         pass
 
@@ -47,10 +47,6 @@ def test_get_color_space(color_space_string, expected):
     [
         (RESOURCE_PATH / "valid_tiff.tif", ""),
         (RESOURCE_PATH / "image5x6x7.mhd", "Image isn't a TIFF file"),
-        (
-            RESOURCE_PATH / "invalid_meta_data_tiff.tif",
-            "Image contains unauthorized information",
-        ),
         (
             RESOURCE_PATH / "invalid_resolutions_tiff.tif",
             "Image only has a single resolution level",
@@ -65,7 +61,7 @@ def test_tiff_validation(resource, expected_error_message):
     error_message = ""
 
     try:
-        load_tiff_file(path=resource)
+        load_tiff_file(resource)
     except ValidationError as e:
         error_message = str(e)
 
@@ -79,11 +75,6 @@ def test_tiff_validation(resource, expected_error_message):
     [
         (RESOURCE_PATH, "valid_tiff.tif", ""),
         (RESOURCE_PATH, "image5x6x7.mhd", "Image isn't a TIFF file"),
-        (
-            RESOURCE_PATH,
-            "invalid_meta_data_tiff.tif",
-            "Image contains unauthorized information",
-        ),
         (
             RESOURCE_PATH,
             "invalid_resolutions_tiff.tif",
@@ -106,8 +97,8 @@ def test_dzi_creation(
 
     try:
         # Load the tiff file and create dzi
-        tiff_file = load_tiff_file(path=temp_file)
-        create_dzi_images(tiff_file=tiff_file, pk=uuid4())
+        tiff_file = load_tiff_file(temp_file)
+        _create_dzi_images(gc_file=tiff_file, pk=uuid4())
     except ValidationError as e:
         error_message = str(e)
 
@@ -135,8 +126,8 @@ def test_tiff_image_entry_creation(
     image_entry = None
     pk = uuid4()
     try:
-        tiff_file = load_tiff_file(path=resource)
-        image_entry = create_tiff_image_entry(tiff_file=tiff_file, pk=pk)
+        tiff_file = load_tiff_file(resource)
+        image_entry = _create_tiff_image_entry(tiff_file=tiff_file, pk=pk)
     except ValidationError as e:
         error_message = str(e)
 
@@ -155,8 +146,10 @@ def test_tiff_image_entry_creation(
         assert image_entry.height == tiff_tags["ImageLength"].value
         assert image_entry.depth == 1
         assert image_entry.resolution_levels == len(tiff_file.pages)
-        assert image_entry.color_space == get_color_space(
-            str(tiff_tags["PhotometricInterpretation"].value)
+        assert image_entry.color_space == _get_color_space(
+            color_space_string=str(
+                tiff_tags["PhotometricInterpretation"].value
+            )
         )
         assert image_entry.voxel_width_mm == approx(voxel_size[0])
         assert image_entry.voxel_height_mm == approx(voxel_size[1])
@@ -173,7 +166,7 @@ def test_image_builder_tiff(tmpdir_factory):
         RESOURCE_PATH, temp_dir, ignore=shutil.ignore_patterns("dicom*")
     )
 
-    image_builder_result = image_builder_tiff(temp_dir)
+    image_builder_result = image_builder_tiff(path=temp_dir)
 
     # Assumes the RESOURCE_PATH folder only contains a single correct TIFF file
     assert len(image_builder_result.consumed_files) == 1