Repository: kserve/kserve

Issue:
kserve does not unpack downloaded zip/tar files from hdfs
/kind feature

**Describe the solution you'd like**
Currently, zip/tar file downloaded from the HDFS is not uncompressed. Adding the changes to uncompress the zip/tar files to be consistent with S3, OSS etc.
So to be consistent with S3, OSS etc we need to unpack archive file incase of HDFS.



Discussion:

Resolving patch:
diff --git a/python/kserve/kserve/storage/storage.py b/python/kserve/kserve/storage/storage.py
--- a/python/kserve/kserve/storage/storage.py
+++ b/python/kserve/kserve/storage/storage.py
@@ -15,26 +15,26 @@
 import base64
 import glob
 import gzip
+import json
 import logging
 import mimetypes
 import os
 import re
-import json
 import shutil
 import tarfile
 import tempfile
-from typing import Dict
 import zipfile
+from pathlib import Path
+from typing import Dict
 from urllib.parse import urlparse
+
+import boto3
 import requests
-from pathlib import Path
 from azure.storage.blob import BlobServiceClient
 from azure.storage.blob._list_blobs_helper import BlobPrefix
 from azure.storage.fileshare import ShareServiceClient
-
-from botocore.client import Config
 from botocore import UNSIGNED
-import boto3
+from botocore.client import Config
 from google.auth import exceptions
 from google.cloud import storage
 
@@ -171,7 +171,7 @@ def _download_s3(uri, temp_dir: str):
         bucket_name = parsed.netloc
         bucket_path = parsed.path.lstrip('/')
 
-        count = 0
+        file_count = 0
         exact_obj_found = False
         bucket = s3.Bucket(bucket_name)
         for obj in bucket.objects.filter(Prefix=bucket_path):
@@ -211,17 +211,17 @@ def _download_s3(uri, temp_dir: str):
                 os.makedirs(os.path.dirname(target), exist_ok=True)
             bucket.download_file(obj.key, target)
             logging.info('Downloaded object %s to %s' % (obj.key, target))
-            count = count + 1
+            file_count += 1
 
             # If the exact object is found, then it is sufficient to download that and break the loop
             if exact_obj_found:
                 break
-        if count == 0:
+        if file_count == 0:
             raise RuntimeError(
                 "Failed to fetch model. No model found in %s." % bucket_path)
 
         # Unpack compressed file, supports .tgz, tar.gz and zip file formats.
-        if count == 1:
+        if file_count == 1:
             mimetype, _ = mimetypes.guess_type(target)
             if mimetype in ["application/x-tar", "application/zip"]:
                 Storage._unpack_archive_file(target, mimetype, temp_dir)
@@ -240,7 +240,7 @@ def _download_gcs(uri, temp_dir: str):
         if not prefix.endswith("/"):
             prefix = prefix + "/"
         blobs = bucket.list_blobs(prefix=prefix)
-        count = 0
+        file_count = 0
         for blob in blobs:
             # Replace any prefix from the object key with temp_dir
             subdir_object_key = blob.name.replace(bucket_path, "", 1).lstrip("/")
@@ -254,13 +254,13 @@ def _download_gcs(uri, temp_dir: str):
                 dest_path = os.path.join(temp_dir, subdir_object_key)
                 logging.info("Downloading: %s", dest_path)
                 blob.download_to_filename(dest_path)
-            count = count + 1
-        if count == 0:
+            file_count += 1
+        if file_count == 0:
             raise RuntimeError(
                 "Failed to fetch model. No model found in %s." % uri)
 
         # Unpack compressed file, supports .tgz, tar.gz and zip file formats.
-        if count == 1:
+        if file_count == 1:
             mimetype, _ = mimetypes.guess_type(blob.name)
             if mimetype in ["application/x-tar", "application/zip"]:
                 Storage._unpack_archive_file(dest_path, mimetype, temp_dir)
@@ -353,6 +353,8 @@ def _download_hdfs(uri, out_dir: str):
                 root=config["HDFS_ROOTPATH"],
                 session=s
             )
+        file_count = 0
+        dest_file_path = ""
 
         # Check path exists and get path status
         # Raises HdfsError when path does not exist
@@ -360,10 +362,20 @@ def _download_hdfs(uri, out_dir: str):
 
         if status["type"] == "FILE":
             client.download(path, out_dir, n_threads=1)
+            file_count += 1
+            file_name = path.rsplit("/", 1)[-1]
+            dest_file_path = f"{out_dir}/{file_name}"
         else:
             files = client.list(path)
+            file_count += len(files)
             for f in files:
                 client.download(f"{path}/{f}", out_dir, n_threads=int(config["N_THREADS"]))
+                dest_file_path = f"{out_dir}/{f}"
+
+        if file_count == 1:
+            mimetype, _ = mimetypes.guess_type(dest_file_path)
+            if mimetype in ["application/x-tar", "application/zip"]:
+                Storage._unpack_archive_file(dest_file_path, mimetype, out_dir)
 
     @staticmethod
     def _download_azure_blob(uri, out_dir: str):  # pylint: disable=too-many-locals
@@ -378,7 +390,7 @@ def _download_azure_blob(uri, out_dir: str):  # pylint: disable=too-many-locals
 
         blob_service_client = BlobServiceClient(account_url, credential=token)
         container_client = blob_service_client.get_container_client(container_name)
-        count = 0
+        file_count = 0
         blobs = []
         max_depth = 5
         stack = [(prefix, max_depth)]
@@ -387,7 +399,7 @@ def _download_azure_blob(uri, out_dir: str):  # pylint: disable=too-many-locals
             if depth < 0:
                 continue
             for item in container_client.walk_blobs(
-                            name_starts_with=curr_prefix):
+                    name_starts_with=curr_prefix):
                 if isinstance(item, BlobPrefix):
                     stack.append((item.name, depth - 1))
                 else:
@@ -400,13 +412,13 @@ def _download_azure_blob(uri, out_dir: str):  # pylint: disable=too-many-locals
             downloader = container_client.download_blob(blob.name)
             with open(dest_path, "wb+") as f:
                 f.write(downloader.readall())
-            count = count + 1
-        if count == 0:
+            file_count += 1
+        if file_count == 0:
             raise RuntimeError(
                 "Failed to fetch model. No model found in %s." % (uri))
 
         # Unpack compressed file, supports .tgz, tar.gz and zip file formats.
-        if count == 1:
+        if file_count == 1:
             mimetype, _ = mimetypes.guess_type(dest_path)
             if mimetype in ["application/x-tar", "application/zip"]:
                 Storage._unpack_archive_file(dest_path, mimetype, out_dir)
@@ -424,7 +436,7 @@ def _download_azure_file_share(uri, out_dir: str):  # pylint: disable=too-many-l
 
         share_service_client = ShareServiceClient(account_url, credential=access_key)
         share_client = share_service_client.get_share_client(share_name)
-        count = 0
+        file_count = 0
         share_files = []
         max_depth = 5
         stack = [(prefix, max_depth)]
@@ -449,13 +461,13 @@ def _download_azure_file_share(uri, out_dir: str):  # pylint: disable=too-many-l
             with open(dest_path, "wb+") as f:
                 data = file_client.download_file()
                 data.readinto(f)
-            count = count + 1
-        if count == 0:
+            file_count += 1
+        if file_count == 0:
             raise RuntimeError(
                 "Failed to fetch model. No model found in %s." % (uri))
 
         # Unpack compressed file, supports .tgz, tar.gz and zip file formats.
-        if count == 1:
+        if file_count == 1:
             mimetype, _ = mimetypes.guess_type(dest_path)
             if mimetype in ["application/x-tar", "application/zip"]:
                 Storage._unpack_archive_file(dest_path, mimetype, out_dir)
@@ -514,7 +526,7 @@ def _download_local(uri, out_dir=None):
         if os.path.isdir(local_path):
             local_path = os.path.join(local_path, "*")
 
-        count = 0
+        file_count = 0
         for src in glob.glob(local_path):
             _, tail = os.path.split(src)
             dest_path = os.path.join(out_dir, tail)
@@ -523,12 +535,12 @@ def _download_local(uri, out_dir=None):
                 os.symlink(src, dest_path)
             else:
                 logging.info("File %s already exist", dest_path)
-            count = count + 1
-        if count == 0:
+            file_count += 1
+        if file_count == 0:
             raise RuntimeError(
                 "Failed to fetch model. No model found in %s." % (uri))
         # Unpack compressed file, supports .tgz, tar.gz and zip file formats.
-        if count == 1:
+        if file_count == 1:
             mimetype, _ = mimetypes.guess_type(dest_path)
             if mimetype in ["application/x-tar", "application/zip"]:
                 Storage._unpack_archive_file(dest_path, mimetype, out_dir)
@@ -560,12 +572,12 @@ def _download_from_uri(uri, out_dir=None):
             if response.status_code != 200:
                 raise RuntimeError("URI: %s returned a %s response code." % (uri, response.status_code))
             zip_content_types = ('application/x-zip-compressed', 'application/zip', 'application/zip-compressed')
-            if mimetype == 'application/zip' and not response.headers.get('Content-Type', '')\
+            if mimetype == 'application/zip' and not response.headers.get('Content-Type', '') \
                     .startswith(zip_content_types):
                 raise RuntimeError("URI: %s did not respond with any of following \'Content-Type\': " % uri +
                                    ", ".join(zip_content_types))
             tar_content_types = ('application/x-tar', 'application/x-gtar', 'application/x-gzip', 'application/gzip')
-            if mimetype == 'application/x-tar' and not response.headers.get('Content-Type', '')\
+            if mimetype == 'application/x-tar' and not response.headers.get('Content-Type', '') \
                     .startswith(tar_content_types):
                 raise RuntimeError("URI: %s did not respond with any of following \'Content-Type\': " % uri +
                                    ", ".join(tar_content_types))


Test patch: