Repository: comic/grand-challenge.org

Issue:
support uploading images and annotations in zipped format
currently uploading images and annotation to a dataset requires a user to select files to upload from the pop-up window. Consider if the number of files to be uploaded is high, one has to track which are already uploaded, a sort of work on trunking manually,  which is cumbersome.
please provide the function that user can upload a single packed file in zip format, so as to maintain data integrity. 


Discussion:

Resolving patch:
diff --git a/app/grandchallenge/cases/tasks.py b/app/grandchallenge/cases/tasks.py
--- a/app/grandchallenge/cases/tasks.py
+++ b/app/grandchallenge/cases/tasks.py
@@ -1,4 +1,9 @@
+import fileinput
+import os
+import re
 import shutil
+import tarfile
+import zipfile
 from pathlib import Path
 from tempfile import mkdtemp
 from typing import Sequence, Tuple
@@ -182,6 +187,115 @@ def remove_duplicate_files(
     )
 
 
+def fix_mhd_file(file, prefix):
+    """
+    Fixes the reference to the data file for extracted mhd files in place.
+
+    This is needed because we prepend folder names to extraced files to prevent
+    duplicate file names.
+
+    Parameters
+    ----------
+    file:
+        A .mhd file.
+    prefix:
+        The prefix that was added to the file and should be added to the
+        ElementDataFile entry.
+    """
+    try:
+        with fileinput.input(file, inplace=True) as f:
+            for line in f:
+                new_line = re.sub(
+                    r"(ElementDataFile)\s+=\s+(.*)", fr"\1 = {prefix}\2", line,
+                )
+                print(new_line, end="")
+    except Exception:
+        pass
+
+
+def extract_and_flatten(file, path, prefix=None, is_tar=False):
+    """
+    Extracts a flattened list of all files in `file` to `path`.
+
+    Parameters
+    ----------
+    file:
+        A zip or tar file.
+    path:
+        The path to which the contents of `file` are to be extracted.
+
+    Returns
+    -------
+    A list of extracted files.
+    """
+    new_files = []
+    listfunc = file.getmembers if is_tar else file.infolist
+    filename_attr = "name" if is_tar else "filename"
+    is_dir_func = "isdir" if is_tar else "is_dir"
+    for info in listfunc():
+        filename = getattr(info, filename_attr)
+        # Skip directories
+        if getattr(info, is_dir_func)():
+            continue
+        # For any file that is inside a directory, prepend the directory
+        # name(s) to the filename
+        _filename = re.sub(r"[/:?]", "-", filename)
+        base_name = os.path.basename(filename)
+        setattr(info, filename_attr, (prefix or "") + _filename)
+        file.extract(info, path)
+        filename = getattr(info, filename_attr)
+        new_files.append(
+            {
+                "prefix": filename.replace(base_name, ""),
+                "path": path / filename,
+            }
+        )
+    return new_files
+
+
+def check_compressed_and_extract(file_path, target_path, prefix=None):
+    """
+    Checks if `file_path` is a zip or tar file and if so, extracts it.
+
+    Parameters
+    ----------
+    file_path:
+        The file path to be checked and possibly extracted.
+    target_path:
+        The path to which the contents of `file_path` are to be extracted.
+    prefix, optional:
+        For compressed files containing a nested structure, the folder names of
+        all containing folders are prepended to the filename. This is done to
+        be able to flatten the contents of the compressed file, without
+        completely losing the information onthe original compressed folder's
+        structure.
+    """
+    new_files = []
+    if tarfile.is_tarfile(file_path):
+        with tarfile.TarFile(file_path) as tf:
+            new_files = extract_and_flatten(
+                tf, target_path, prefix=prefix, is_tar=True
+            )
+    elif zipfile.is_zipfile(file_path):
+        with zipfile.ZipFile(file_path) as zf:
+            new_files = extract_and_flatten(zf, target_path, prefix=prefix)
+    # is_tarfile seems to recognize non-tarfiles as tarfiles, so check
+    # if anything has been processed before removing the file.
+    if new_files:
+        file_path.unlink()
+    for file in new_files:
+        if file["path"].name.endswith(".mhd") and file["prefix"]:
+            fix_mhd_file(file["path"], file["prefix"])
+        check_compressed_and_extract(
+            file["path"], target_path, prefix=file["prefix"]
+        )
+
+
+def extract_files(source_path: Path):
+    for file_path in source_path.iterdir():
+        check_compressed_and_extract(file_path, source_path)
+
+
 @shared_task  # noqa: C901
 def build_images(upload_session_uuid: UUID):
     """
@@ -237,10 +351,17 @@ def build_images(upload_session_uuid: UUID):
 
                 populate_provisioning_directory(session_files, tmp_dir)
 
+                extract_files(tmp_dir)
+                session_files = [
+                    RawImageFile.objects.get_or_create(
+                        filename=file.name, upload_session=upload_session,
+                    )[0]
+                    for file in tmp_dir.iterdir()
+                ]
                 filename_lookup = {
-                    StagedAjaxFile(
-                        raw_image_file.staged_file_id
-                    ).name: raw_image_file
+                    raw_image_file.staged_file_id
+                    and StagedAjaxFile(raw_image_file.staged_file_id).name
+                    or raw_image_file.filename: raw_image_file
                     for raw_image_file in session_files
                 }
                 unconsumed_filenames = set(filename_lookup.keys())
@@ -341,9 +462,10 @@ def build_images(upload_session_uuid: UUID):
                 # Delete any touched file data
                 for file in session_files:
                     try:
-                        saf = StagedAjaxFile(file.staged_file_id)
-                        file.staged_file_id = None
-                        saf.delete()
+                        if file.staged_file_id:
+                            saf = StagedAjaxFile(file.staged_file_id)
+                            file.staged_file_id = None
+                            saf.delete()
                         file.save()
                     except NotFoundError:
                         pass


Test patch:
diff --git a/app/tests/cases_tests/resources/test.tar b/app/tests/cases_tests/resources/test.tar
new file mode 100644
Binary files /dev/null and b/app/tests/cases_tests/resources/test.tar differ
diff --git a/app/tests/cases_tests/resources/test.zip b/app/tests/cases_tests/resources/test.zip
new file mode 100644
Binary files /dev/null and b/app/tests/cases_tests/resources/test.zip differ
diff --git a/app/tests/cases_tests/resources/valid.zip b/app/tests/cases_tests/resources/valid.zip
new file mode 100644
Binary files /dev/null and b/app/tests/cases_tests/resources/valid.zip differ
diff --git a/app/tests/cases_tests/test_background_tasks.py b/app/tests/cases_tests/test_background_tasks.py
--- a/app/tests/cases_tests/test_background_tasks.py
+++ b/app/tests/cases_tests/test_background_tasks.py
@@ -1,3 +1,8 @@
+import re
+import shutil
+import tarfile
+import zipfile
+from pathlib import Path
 from typing import Dict, List, Tuple
 
 import SimpleITK
@@ -15,6 +20,11 @@
     RawImageUploadSession,
     UploadSessionState,
 )
+from grandchallenge.cases.tasks import (
+    check_compressed_and_extract,
+    extract_and_flatten,
+    fix_mhd_file,
+)
 from grandchallenge.jqfileupload.widgets.uploader import StagedAjaxFile
 from tests.cases_tests import RESOURCE_PATH
 from tests.factories import UserFactory
@@ -291,3 +301,109 @@ def test_mhd_file_annotation_creation(settings):
 
     sitk_image = image.get_sitk_image()
     assert [e for e in reversed(sitk_image.GetSize())] == image.shape
+
+
+def test_fix_mhd_file(tmpdir):
+    file = RESOURCE_PATH / "image3x4.mhd"
+    tmp_file = shutil.copy(str(file), str(tmpdir))
+    tmp_file = Path(tmp_file)
+    with tmp_file.open("r") as f:
+        headers = f.read()
+    headers = re.match(
+        r".*ElementDataFile = (?P<data_file>[^\n]*)", headers, flags=re.DOTALL,
+    )
+    assert headers.group("data_file") == "image3x4.zraw"
+
+    fix_mhd_file(tmp_file, "foo-")
+    with tmp_file.open("r") as f:
+        headers = f.read()
+    headers = re.match(
+        r".*ElementDataFile = (?P<data_file>[^\n]*)", headers, flags=re.DOTALL,
+    )
+    assert headers.group("data_file") == "foo-image3x4.zraw"
+
+
+@pytest.mark.parametrize(
+    "file_name,func,is_tar",
+    (
+        ("test.zip", zipfile.ZipFile, False),
+        ("test.tar", tarfile.TarFile, True),
+    ),
+)
+def test_extract_and_flatten(tmpdir, file_name, func, is_tar):
+    file = RESOURCE_PATH / file_name
+    tmp_file = shutil.copy(str(file), str(tmpdir))
+    tmp_file = Path(tmp_file)
+    assert tmpdir.listdir() == [tmp_file]
+
+    tmpdir_path = Path(tmpdir)
+    with func(tmp_file) as f:
+        new_files = extract_and_flatten(f, tmpdir_path, is_tar=is_tar)
+
+    expected = [
+        {"prefix": "folder-0-", "path": tmpdir_path / "folder-0-file-0.txt"},
+        {
+            "prefix": "folder-0-folder-1-",
+            "path": tmpdir_path / "folder-0-folder-1-file-1.txt",
+        },
+        {
+            "prefix": "folder-0-folder-1-folder-2-",
+            "path": tmpdir_path / "folder-0-folder-1-folder-2-file-2.txt",
+        },
+        {
+            "prefix": "folder-0-folder-1-folder-2-",
+            "path": tmpdir_path / "folder-0-folder-1-folder-2-folder-3.zip",
+        },
+    ]
+    assert sorted(new_files, key=lambda k: k["path"]) == expected
+    assert sorted(tmpdir.listdir()) == sorted(
+        [x["path"] for x in expected] + [tmpdir_path / file_name]
+    )
+
+
+@pytest.mark.parametrize("file_name", ("test.zip", "test.tar"))
+def test_check_compressed_and_extract(tmpdir, file_name):
+    file = RESOURCE_PATH / file_name
+    tmp_file = shutil.copy(str(file), str(tmpdir))
+    tmp_file = Path(tmp_file)
+    assert tmpdir.listdir() == [tmp_file]
+
+    tmpdir_path = Path(tmpdir)
+    check_compressed_and_extract(tmp_file, tmpdir_path)
+
+    expected = [
+        "folder-0-file-0.txt",
+        "folder-0-folder-1-file-1.txt",
+        "folder-0-folder-1-folder-2-file-2.txt",
+        "folder-0-folder-1-folder-2-folder-3-file-3.txt",
+    ]
+
+    assert sorted([x.name for x in tmpdir_path.iterdir()]) == expected
+
+
+@pytest.mark.django_db
+def test_build_zip_file(settings):
+    settings.task_eager_propagates = (True,)
+    settings.task_always_eager = (True,)
+
+    # valid.zip contains a tarred version of the dicom folder,
+    # image10x10x10.[mha,mhd,zraw] and valid_tiff.tiff
+    images = ["valid.zip"]
+    session, uploaded_images = create_raw_upload_image_session(images)
+
+    session.refresh_from_db()
+    assert session.error_message is None
+    images = session.image_set.all()
+    assert images.count() == 4
+    # image10x10x10.mha image10x10x10.[mhd,zraw]
+    assert (
+        len([x for x in images if x.shape_without_color == [10, 10, 10]]) == 2
+    )
+    # dicom.tar
+    assert (
+        len([x for x in images if x.shape_without_color == [19, 4, 2, 3]]) == 1
+    )
+    # valid_tiff.tiff
+    assert (
+        len([x for x in images if x.shape_without_color == [1, 205, 205]]) == 1
+    )