Repository: comic/grand-challenge.org

Issue:
Multi-zip uploads fail when content filepaths are not unique



Discussion:

Resolving patch:
diff --git a/app/grandchallenge/cases/tasks.py b/app/grandchallenge/cases/tasks.py
--- a/app/grandchallenge/cases/tasks.py
+++ b/app/grandchallenge/cases/tasks.py
@@ -229,7 +229,7 @@ def fix_mhd_file(file, prefix):
         pass
 
 
-def extract_and_flatten(file, path, prefix="", is_tar=False):
+def extract_and_flatten(file, path, index, prefix="", is_tar=False):
     """
     Extracts a flattened list of all files in `file` to `path`.
 
@@ -248,7 +248,8 @@ def extract_and_flatten(file, path, prefix="", is_tar=False):
     listfunc = file.getmembers if is_tar else file.infolist
     filename_attr = "name" if is_tar else "filename"
     is_dir_func = "isdir" if is_tar else "is_dir"
-    for info in listfunc():
+    indices = {}
+    for info in sorted(listfunc(), key=lambda k: getattr(k, filename_attr)):
         filename = getattr(info, filename_attr)
         # Skip directories
         if getattr(info, is_dir_func)():
@@ -257,10 +258,13 @@ def extract_and_flatten(file, path, prefix="", is_tar=False):
         # name(s) to the filename
         _filename = "-".join(Path(filename).parts[-2:])
         base_name = os.path.basename(filename)
+        folder = filename.replace(base_name, "")
+        if folder:
+            indices[folder] = indices.get(folder, index + 1)
+            index = indices[folder]
         _prefix = _filename.replace(base_name, "")
-        setattr(
-            info, filename_attr, (prefix if not _prefix else "") + _filename
-        )
+        prefix = f"{index}-{prefix if not _prefix else ''}"
+        setattr(info, filename_attr, prefix + _filename)
         file.extract(info, path)
         filename = getattr(info, filename_attr)
         new_files.append(
@@ -269,10 +273,10 @@ def extract_and_flatten(file, path, prefix="", is_tar=False):
                 "path": path / filename,
             }
         )
-    return new_files
+    return max(indices.values()), new_files
 
 
-def check_compressed_and_extract(file_path, target_path, prefix=None):
+def check_compressed_and_extract(file_path, target_path, index, prefix=None):
     """
     Checks if `file_path` is a zip or tar file and if so, extracts it.
 
@@ -292,27 +296,34 @@ def check_compressed_and_extract(file_path, target_path, prefix=None):
     new_files = []
     if tarfile.is_tarfile(file_path):
         with tarfile.TarFile(file_path) as tf:
-            new_files = extract_and_flatten(
-                tf, target_path, prefix=prefix, is_tar=True
+            index, new_files = extract_and_flatten(
+                tf, target_path, index, prefix=prefix, is_tar=True
             )
+            # index += 1
     elif zipfile.is_zipfile(file_path):
         with zipfile.ZipFile(file_path) as zf:
-            new_files = extract_and_flatten(zf, target_path, prefix=prefix)
+            index, new_files = extract_and_flatten(
+                zf, target_path, index, prefix=prefix
+            )
+            # index += 1
     # is_tarfile seems to recognize non-tarfiles as tarfiles, so check
     # if anything has been processed before removing the file.
+
     if new_files:
         file_path.unlink()
     for file in new_files:
         if file["path"].name.endswith(".mhd") and file["prefix"]:
             fix_mhd_file(file["path"], file["prefix"])
-        check_compressed_and_extract(
-            file["path"], target_path, prefix=file["prefix"]
+        index = check_compressed_and_extract(
+            file["path"], target_path, index, prefix=file["prefix"]
         )
+    return index
 
 
 def extract_files(source_path: Path):
+    index = 0
     for file_path in source_path.iterdir():
-        check_compressed_and_extract(file_path, source_path)
+        index = check_compressed_and_extract(file_path, source_path, index)
 
 
 @shared_task


Test patch:
diff --git a/app/tests/cases_tests/resources/same_name.zip b/app/tests/cases_tests/resources/same_name.zip
new file mode 100644
Binary files /dev/null and b/app/tests/cases_tests/resources/same_name.zip differ
diff --git a/app/tests/cases_tests/resources/same_name_zipped.zip b/app/tests/cases_tests/resources/same_name_zipped.zip
new file mode 100644
Binary files /dev/null and b/app/tests/cases_tests/resources/same_name_zipped.zip differ
diff --git a/app/tests/cases_tests/test_background_tasks.py b/app/tests/cases_tests/test_background_tasks.py
--- a/app/tests/cases_tests/test_background_tasks.py
+++ b/app/tests/cases_tests/test_background_tasks.py
@@ -343,15 +343,26 @@ def test_extract_and_flatten(tmpdir, file_name, func, is_tar):
 
     tmpdir_path = Path(tmpdir)
     with func(tmp_file) as f:
-        new_files = extract_and_flatten(f, tmpdir_path, is_tar=is_tar)
-
+        index, new_files = extract_and_flatten(
+            f, tmpdir_path, 0, is_tar=is_tar
+        )
+    assert index == 3
     expected = [
-        {"prefix": "folder-0-", "path": tmpdir_path / "folder-0-file-0.txt"},
-        {"prefix": "folder-1-", "path": tmpdir_path / "folder-1-file-1.txt"},
-        {"prefix": "folder-2-", "path": tmpdir_path / "folder-2-file-2.txt"},
         {
-            "prefix": "folder-2-",
-            "path": tmpdir_path / "folder-2-folder-3.zip",
+            "prefix": "1-folder-0-",
+            "path": tmpdir_path / "1-folder-0-file-0.txt",
+        },
+        {
+            "prefix": "2-folder-1-",
+            "path": tmpdir_path / "2-folder-1-file-1.txt",
+        },
+        {
+            "prefix": "3-folder-2-",
+            "path": tmpdir_path / "3-folder-2-file-2.txt",
+        },
+        {
+            "prefix": "3-folder-2-",
+            "path": tmpdir_path / "3-folder-2-folder-3.zip",
         },
     ]
     assert sorted(new_files, key=lambda k: k["path"]) == expected
@@ -368,17 +379,39 @@ def test_check_compressed_and_extract(tmpdir, file_name):
     assert tmpdir.listdir() == [tmp_file]
 
     tmpdir_path = Path(tmpdir)
-    check_compressed_and_extract(tmp_file, tmpdir_path)
+    check_compressed_and_extract(tmp_file, tmpdir_path, 0)
 
     expected = [
-        "folder-0-file-0.txt",
-        "folder-1-file-1.txt",
-        "folder-2-file-2.txt",
-        "folder-3-file-3.txt",
+        "1-folder-0-file-0.txt",
+        "2-folder-1-file-1.txt",
+        "3-folder-2-file-2.txt",
+        "4-folder-3-file-3.txt",
     ]
     assert sorted([x.name for x in tmpdir_path.iterdir()]) == expected
 
 
+@pytest.mark.parametrize(
+    "file_name,add_one",
+    (("same_name.zip", False), ("same_name_zipped.zip", True)),
+)
+def test_check_compressed_and_extract_same_name(tmpdir, file_name, add_one):
+    file = RESOURCE_PATH / file_name
+    tmp_file = shutil.copy(str(file), str(tmpdir))
+    tmp_file = Path(tmp_file)
+    assert tmpdir.listdir() == [tmp_file]
+
+    tmpdir_path = Path(tmpdir)
+    check_compressed_and_extract(tmp_file, tmpdir_path, 0)
+
+    expected = sorted(
+        [
+            f"{x + 1 if add_one else x}-1-test_grayscale.png"
+            for x in range(1, 11)
+        ]
+    )
+    assert sorted([x.name for x in tmpdir_path.iterdir()]) == expected
+
+
 @pytest.mark.django_db
 def test_build_zip_file(settings):
     settings.task_eager_propagates = (True,)