Repository: kserve/kserve

Issue:
Support download model through general WebHDFS REST API 
/kind feature

**Describe the solution you'd like**
[A clear and concise description of what you want to happen.]

We have a requirement of downloading file from our Hadoop based storage using WebHDFS REST API with TLS (has to provide TLS cert and a header with id). To make it general, TLS and headers can be optional.


User can use the following command to generate the secret:
```sh
kubectl create secret generic hdfscreds \
    --from-file=TLS_CERT=./client.crt \
    --from-file=TLS_KEY=./client.key \
    --from-literal=HDFS_NAMENODE="https://host:port" \
    --from-literal=HDFS_ROOTPATH="/" \
    --from-literal=HEADERS="{'x-my-container': 'my-container'}"
```

format of the secret:
```yaml
apiVersion: v1
kind: Secret
metadata:
  name: hdfscreds
type: Opaque
data:
  HDFS_NAMENODE: 'http(s)://host:port'
  HDFS_ROOTPATH: xxxx # string, default: "/"
  TLS_CERT: xxxx # string, default: ""
  TLS_KEY: xxxx # string, default: ""
  TLS_CA: xxxx # string, default: ""
  TLS_INSECURE_SKIP_VERIFY: # string (true|false), default: "false"
  HEADERS: xxxx # string, default: ""
```

To implement it in storage initializer, [hdfscli](https://hdfscli.readthedocs.io/en/latest/quickstart.html#python-bindings) is used.

simple code sample:

```python
from requests import Session
from hdfs.client import Client

HDFS_NAMENODE="https://host:port"
HDFS_ROOTPATH="/"
HEADERS={'x-my-container': 'my-container'} # parse from string
TLS_CERT="path to TLS_CERT"
TLS_KEY="path to TLS_KEY"
TLS_CA="path to TLS_CA"
TLS_INSECURE_SKIP_VERIFY = True|False
FILE_PATH = "path to file or directory"
OUT_PATH = "path to output directory"


s = Session()
s.cert = (TLS_CERT, TLS_KEY)
# s.verify = , True, False, or CA PATH
if TLS_CA:
  s.verify = TLS_CA
if TLS_INSECURE_SKIP_VERIFY:
  s.verify = False

s.headers.update(HEADERS)

# Has to set root to none empty value otherwise it will try to GETHOMEDIRECTORY, but user don't have the permission
client = Client(url=HDFS_NAMENODE, root=HDFS_ROOTPATH, session=s)
status = client.status(FILE_PATH, strict=False)
print(status) # None of not exist

# Downlaod file or directory to output directory
client.download(FILE_PATH, OUT_PATH, n_threads=2)
```

The TLS_CERT and TLS_KEY as well as TLS_CA can be mounted as volume just like GCS or as a temp file if it is supported.


**Anything else you would like to add:**
[Miscellaneous information that will assist in solving the issue.]

https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html
https://hdfscli.readthedocs.io/en/latest/quickstart.html#python-bindings


Discussion:

Resolving patch:
diff --git a/python/kserve/kserve/storage.py b/python/kserve/kserve/storage.py
--- a/python/kserve/kserve/storage.py
+++ b/python/kserve/kserve/storage.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import glob
 import gzip
 import logging
@@ -22,6 +23,7 @@
 import shutil
 import tarfile
 import tempfile
+from typing import Dict
 import zipfile
 from urllib.parse import urlparse
 import requests
@@ -40,6 +42,8 @@
 
 _GCS_PREFIX = "gs://"
 _S3_PREFIX = "s3://"
+_HDFS_PREFIX = "hdfs://"
+_WEBHDFS_PREFIX = "webhdfs://"
 _AZURE_BLOB_RE = "https://(.+?).blob.core.windows.net/(.+)"
 _AZURE_FILE_RE = "https://(.+?).file.core.windows.net/(.+)"
 _LOCAL_PREFIX = "file://"
@@ -48,11 +52,14 @@
 _HEADERS_SUFFIX = "-headers"
 _PVC_PREFIX = "/mnt/pvc"
 
+_HDFS_SECRET_DIRECTORY = "/var/secrets/kserve-hdfscreds"
+_HDFS_FILE_SECRETS = ["KERBEROS_KEYTAB", "TLS_CERT", "TLS_KEY", "TLS_CA"]
+
 
 class Storage(object):  # pylint: disable=too-few-public-methods
     @staticmethod
     def download(uri: str, out_dir: str = None) -> str:
-        uri = Storage._update_with_storage_spec(uri)
+        Storage._update_with_storage_spec()
         logging.info("Copying contents of %s to local", uri)
 
         if uri.startswith(_PVC_PREFIX) and not os.path.exists(uri):
@@ -74,6 +81,8 @@ def download(uri: str, out_dir: str = None) -> str:
             Storage._download_gcs(uri, out_dir)
         elif uri.startswith(_S3_PREFIX):
             Storage._download_s3(uri, out_dir)
+        elif uri.startswith(_HDFS_PREFIX) or uri.startswith(_WEBHDFS_PREFIX):
+            Storage._download_hdfs(uri, out_dir)
         elif re.search(_AZURE_BLOB_RE, uri):
             Storage._download_azure_blob(uri, out_dir)
         elif re.search(_AZURE_FILE_RE, uri):
@@ -95,21 +104,35 @@ def download(uri: str, out_dir: str = None) -> str:
         return out_dir
 
     @staticmethod
-    def _update_with_storage_spec(uri: str) -> str:
+    def _update_with_storage_spec():
         storage_secret_json = json.loads(os.environ.get("STORAGE_CONFIG", "{}"))
         storage_secret_override_params = json.loads(os.environ.get("STORAGE_OVERRIDE_CONFIG", "{}"))
         if storage_secret_override_params:
             for key, value in storage_secret_override_params.items():
                 storage_secret_json[key] = value
+
         if storage_secret_json.get("type", "") == "s3":
-            uri = uri.replace("<bucket-placeholder>", storage_secret_json.get("bucket", ""))
             os.environ["AWS_ENDPOINT_URL"] = storage_secret_json.get("endpoint_url", "")
             os.environ["AWS_ACCESS_KEY_ID"] = storage_secret_json.get("access_key_id", "")
             os.environ["AWS_SECRET_ACCESS_KEY"] = storage_secret_json.get("secret_access_key", "")
             os.environ["AWS_DEFAULT_REGION"] = storage_secret_json.get("region", "")
             os.environ["AWS_CA_BUNDLE"] = storage_secret_json.get("certificate", "")
             os.environ["awsAnonymousCredential"] = storage_secret_json.get("anonymous", "")
-        return uri
+
+        if storage_secret_json.get("type", "") == "hdfs" or storage_secret_json.get("type", "") == "webhdfs":
+            temp_dir = tempfile.mkdtemp()
+            os.environ["HDFS_SECRET_DIR"] = temp_dir
+            for key, value in storage_secret_json.items():
+                mode = "w"
+
+                # If the secret is supposed to be a file, then it was base64 encoded in the json
+                if key in _HDFS_FILE_SECRETS:
+                    value = base64.b64decode(value)
+                    mode = "wb"
+
+                with open(f"{temp_dir}/{key}", mode) as f:
+                    f.write(value)
+                    f.flush()
 
     @staticmethod
     def get_S3_config():
@@ -215,6 +238,106 @@ def _download_gcs(uri, temp_dir: str):
             if mimetype in ["application/x-tar", "application/zip"]:
                 Storage._unpack_archive_file(dest_path, mimetype, temp_dir)
 
+    @staticmethod
+    def _load_hdfs_configuration() -> Dict:
+        config = {
+            "HDFS_NAMENODE": None,
+            "USER_PROXY": None,
+            "HDFS_ROOTPATH": None,
+            "TLS_CERT": None,
+            "TLS_KEY": None,
+            "TLS_CA": None,
+            "TLS_SKIP_VERIFY": "false",
+            "HEADERS": None,
+            "N_THREADS": "2",
+            "KERBEROS_KEYTAB": None,
+            "KERBEROS_PRINCIPAL": None,
+        }
+
+        secret_dir = _HDFS_SECRET_DIRECTORY
+        if os.environ.get("HDFS_SECRET_DIR"):
+            secret_dir = os.environ["HDFS_SECRET_DIR"]
+
+        for filename in os.listdir(secret_dir):
+            if filename not in config:
+                continue
+
+            # We don't read files which are supposed to be files, just save their path
+            if filename in _HDFS_FILE_SECRETS:
+                config[filename] = f"{secret_dir}/{filename}"
+                continue
+
+            # Read file and save value in config dict
+            with open(f"{secret_dir}/{filename}") as f:
+                config[filename] = f.read()
+
+        return config
+
+    @staticmethod
+    def _download_hdfs(uri, out_dir: str):
+        from krbcontext.context import krbContext
+        from hdfs.ext.kerberos import Client, KerberosClient
+
+        config = Storage._load_hdfs_configuration()
+
+        logging.info(f"Using the following hdfs config\n{config}")
+
+        # Remove hdfs:// or webhdfs:// from the uri to get just the path
+        # e.g. hdfs://user/me/model -> user/me/model
+        if uri.startswith(_HDFS_PREFIX):
+            path = uri[len(_HDFS_PREFIX):]
+        else:
+            path = uri[len(_WEBHDFS_PREFIX):]
+
+        if not config["HDFS_ROOTPATH"]:
+            path = "/" + path
+
+        s = requests.Session()
+
+        if config["TLS_CERT"]:
+            s.cert = (config["TLS_CERT"], config["TLS_KEY"])
+        # s.verify = , True, False, or CA PATH
+        if config["TLS_CA"]:
+            s.verify = config["TLS_CA"]
+        if config["TLS_SKIP_VERIFY"].lower() == "true":
+            s.verify = False
+
+        if config["HEADERS"]:
+            headers = json.loads(config["HEADERS"])
+            s.headers.update(headers)
+
+        if config["KERBEROS_PRINCIPAL"]:
+            context = krbContext(
+                using_keytab=True,
+                principal=config["KERBEROS_PRINCIPAL"],
+                keytab_file=config["KERBEROS_KEYTAB"]
+            )
+            context.init_with_keytab()
+            client = KerberosClient(
+                config["HDFS_NAMENODE"],
+                proxy=config["USER_PROXY"],
+                root=config["HDFS_ROOTPATH"],
+                session=s
+            )
+        else:
+            client = Client(
+                config["HDFS_NAMENODE"],
+                proxy=config["USER_PROXY"],
+                root=config["HDFS_ROOTPATH"],
+                session=s
+            )
+
+        # Check path exists and get path status
+        # Raises HdfsError when path does not exist
+        status = client.status(path)
+
+        if status["type"] == "FILE":
+            client.download(path, out_dir, n_threads=1)
+        else:
+            files = client.list(path)
+            for f in files:
+                client.download(f"{path}/{f}", out_dir, n_threads=int(config["N_THREADS"]))
+
     @staticmethod
     def _download_azure_blob(uri, out_dir: str):  # pylint: disable=too-many-locals
         account_name, account_url, container_name, prefix = Storage._parse_azure_uri(uri)


Test patch:
diff --git a/pkg/apis/serving/v1beta1/explainer_alibi_test.go b/pkg/apis/serving/v1beta1/explainer_alibi_test.go
--- a/pkg/apis/serving/v1beta1/explainer_alibi_test.go
+++ b/pkg/apis/serving/v1beta1/explainer_alibi_test.go
@@ -69,7 +69,7 @@ func TestAlibiValidation(t *testing.T) {
 			spec: ExplainerSpec{
 				Alibi: &AlibiExplainerSpec{
 					ExplainerExtensionSpec: ExplainerExtensionSpec{
-						StorageURI: "hdfs://modelzoo",
+						StorageURI: "invaliduri://modelzoo",
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/explainer_custom_test.go b/pkg/apis/serving/v1beta1/explainer_custom_test.go
--- a/pkg/apis/serving/v1beta1/explainer_custom_test.go
+++ b/pkg/apis/serving/v1beta1/explainer_custom_test.go
@@ -63,7 +63,7 @@ func TestCustomExplainerValidation(t *testing.T) {
 							Env: []v1.EnvVar{
 								{
 									Name:  "STORAGE_URI",
-									Value: "hdfs://modelzoo",
+									Value: "invaliduri://modelzoo",
 								},
 							},
 						},
diff --git a/pkg/apis/serving/v1beta1/predictor_custom_test.go b/pkg/apis/serving/v1beta1/predictor_custom_test.go
--- a/pkg/apis/serving/v1beta1/predictor_custom_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_custom_test.go
@@ -60,7 +60,7 @@ func TestCustomPredictorValidation(t *testing.T) {
 							Env: []v1.EnvVar{
 								{
 									Name:  "STORAGE_URI",
-									Value: "hdfs://modelzoo",
+									Value: "invaliduri://modelzoo",
 								},
 							},
 						},
diff --git a/pkg/apis/serving/v1beta1/predictor_lightgbm_test.go b/pkg/apis/serving/v1beta1/predictor_lightgbm_test.go
--- a/pkg/apis/serving/v1beta1/predictor_lightgbm_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_lightgbm_test.go
@@ -60,7 +60,7 @@ func TestLightGBMValidation(t *testing.T) {
 			spec: PredictorSpec{
 				LightGBM: &LightGBMSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("hdfs://modelzoo"),
+						StorageURI: proto.String("invaliduri://modelzoo"),
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/predictor_onnxruntime_test.go b/pkg/apis/serving/v1beta1/predictor_onnxruntime_test.go
--- a/pkg/apis/serving/v1beta1/predictor_onnxruntime_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_onnxruntime_test.go
@@ -60,7 +60,7 @@ func TestOnnxRuntimeValidation(t *testing.T) {
 			spec: PredictorSpec{
 				ONNX: &ONNXRuntimeSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("hdfs://modelzoo"),
+						StorageURI: proto.String("invaliduri://modelzoo"),
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/predictor_paddle_test.go b/pkg/apis/serving/v1beta1/predictor_paddle_test.go
--- a/pkg/apis/serving/v1beta1/predictor_paddle_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_paddle_test.go
@@ -59,7 +59,7 @@ func TestPaddleValidation(t *testing.T) {
 			spec: PredictorSpec{
 				Paddle: &PaddleServerSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("hdfs://modelzoo"),
+						StorageURI: proto.String("invaliduri://modelzoo"),
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/predictor_pmml_test.go b/pkg/apis/serving/v1beta1/predictor_pmml_test.go
--- a/pkg/apis/serving/v1beta1/predictor_pmml_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_pmml_test.go
@@ -60,7 +60,7 @@ func TestPMMLValidation(t *testing.T) {
 			spec: PredictorSpec{
 				PMML: &PMMLSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("hdfs://modelzoo"),
+						StorageURI: proto.String("invaliduri://modelzoo"),
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/predictor_sklearn_test.go b/pkg/apis/serving/v1beta1/predictor_sklearn_test.go
--- a/pkg/apis/serving/v1beta1/predictor_sklearn_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_sklearn_test.go
@@ -61,7 +61,7 @@ func TestSKLearnValidation(t *testing.T) {
 			spec: PredictorSpec{
 				SKLearn: &SKLearnSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("hdfs://modelzoo"),
+						StorageURI: proto.String("invaliduri://modelzoo"),
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/predictor_tfserving_test.go b/pkg/apis/serving/v1beta1/predictor_tfserving_test.go
--- a/pkg/apis/serving/v1beta1/predictor_tfserving_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_tfserving_test.go
@@ -85,7 +85,7 @@ func TestTensorflowValidation(t *testing.T) {
 			spec: PredictorSpec{
 				Tensorflow: &TFServingSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("s3://modelzoo"),
+						StorageURI: proto.String("hdfs://modelzoo"),
 					},
 				},
 			},
@@ -95,7 +95,7 @@ func TestTensorflowValidation(t *testing.T) {
 			spec: PredictorSpec{
 				Tensorflow: &TFServingSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("hdfs://modelzoo"),
+						StorageURI: proto.String("invaliduri://modelzoo"),
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/predictor_torchserve_test.go b/pkg/apis/serving/v1beta1/predictor_torchserve_test.go
--- a/pkg/apis/serving/v1beta1/predictor_torchserve_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_torchserve_test.go
@@ -94,7 +94,7 @@ func TestTorchServeValidation(t *testing.T) {
 			spec: PredictorSpec{
 				PyTorch: &TorchServeSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("hdfs://modelzoo"),
+						StorageURI: proto.String("invaliduri://modelzoo"),
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/predictor_triton_test.go b/pkg/apis/serving/v1beta1/predictor_triton_test.go
--- a/pkg/apis/serving/v1beta1/predictor_triton_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_triton_test.go
@@ -60,7 +60,7 @@ func TestTritonValidation(t *testing.T) {
 			spec: PredictorSpec{
 				Triton: &TritonSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("hdfs://modelzoo"),
+						StorageURI: proto.String("invaliduri://modelzoo"),
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/predictor_xgboost_test.go b/pkg/apis/serving/v1beta1/predictor_xgboost_test.go
--- a/pkg/apis/serving/v1beta1/predictor_xgboost_test.go
+++ b/pkg/apis/serving/v1beta1/predictor_xgboost_test.go
@@ -61,7 +61,7 @@ func TestXGBoostValidation(t *testing.T) {
 			spec: PredictorSpec{
 				XGBoost: &XGBoostSpec{
 					PredictorExtensionSpec: PredictorExtensionSpec{
-						StorageURI: proto.String("hdfs://modelzoo"),
+						StorageURI: proto.String("invaliduri://modelzoo"),
 					},
 				},
 			},
diff --git a/pkg/apis/serving/v1beta1/transformer_custom_test.go b/pkg/apis/serving/v1beta1/transformer_custom_test.go
--- a/pkg/apis/serving/v1beta1/transformer_custom_test.go
+++ b/pkg/apis/serving/v1beta1/transformer_custom_test.go
@@ -61,7 +61,7 @@ func TestTransformerValidation(t *testing.T) {
 							Env: []v1.EnvVar{
 								{
 									Name:  "STORAGE_URI",
-									Value: "hdfs://modelzoo",
+									Value: "invaliduri://modelzoo",
 								},
 							},
 						},
diff --git a/pkg/credentials/hdfs/hdfs_secret_test.go b/pkg/credentials/hdfs/hdfs_secret_test.go
new file mode 100644
--- /dev/null
+++ b/pkg/credentials/hdfs/hdfs_secret_test.go
@@ -0,0 +1,76 @@
+/*
+Copyright 2022 The KServe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hdfs
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	secretName = "hdfscreds"
+)
+
+func TestHdfsSecret(t *testing.T) {
+	scenarios := map[string]struct {
+		secret              *v1.Secret
+		expectedVolume      v1.Volume
+		expectedVolumeMount v1.VolumeMount
+	}{
+		"HdfsSecret": {
+			secret: &v1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: secretName,
+				},
+				Data: map[string][]byte{
+					HdfsNamenode:      []byte("https://testdomain:port"),
+					HdfsRootPath:      []byte("/"),
+					KerberosPrincipal: []byte("account@REALM"),
+					KerberosKeytab:    []byte("AAA="),
+				},
+			},
+			expectedVolume: v1.Volume{
+				Name: HdfsVolumeName,
+				VolumeSource: v1.VolumeSource{
+					Secret: &v1.SecretVolumeSource{
+						SecretName: secretName,
+					},
+				},
+			},
+			expectedVolumeMount: v1.VolumeMount{
+				Name:      HdfsVolumeName,
+				ReadOnly:  true,
+				MountPath: MountPath,
+			},
+		},
+	}
+
+	for name, scenario := range scenarios {
+		volume, volumeMount := BuildSecret(scenario.secret)
+
+		if diff := cmp.Diff(scenario.expectedVolume, volume); diff != "" {
+			t.Errorf("Test %q unexpected volume (-want +got): %v", name, diff)
+		}
+
+		if diff := cmp.Diff(scenario.expectedVolumeMount, volumeMount); diff != "" {
+			t.Errorf("Test %q unexpected volumeMount (-want +got): %v", name, diff)
+		}
+	}
+}
diff --git a/pkg/credentials/service_account_credentials_test.go b/pkg/credentials/service_account_credentials_test.go
--- a/pkg/credentials/service_account_credentials_test.go
+++ b/pkg/credentials/service_account_credentials_test.go
@@ -21,15 +21,15 @@ import (
 	"testing"
 
 	"github.com/kserve/kserve/pkg/credentials/azure"
+	"github.com/kserve/kserve/pkg/credentials/gcs"
+  "github.com/kserve/kserve/pkg/credentials/hdfs"
+	"github.com/kserve/kserve/pkg/credentials/s3"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/onsi/gomega"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	knservingv1 "knative.dev/serving/pkg/apis/serving/v1"
-
-	"github.com/kserve/kserve/pkg/credentials/gcs"
-	"github.com/kserve/kserve/pkg/credentials/s3"
 )
 
 var configMap = &v1.ConfigMap{
@@ -421,6 +421,117 @@ func TestAzureCredentialBuilder(t *testing.T) {
 	g.Expect(c.Delete(context.TODO(), customOnlyServiceAccount)).NotTo(gomega.HaveOccurred())
 }
 
+func TestHdfsCredentialBuilder(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	customOnlyServiceAccount := &v1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "custom-sa",
+			Namespace: "default",
+		},
+		Secrets: []v1.ObjectReference{
+			{
+				Name:      "hdfs-custom-secret",
+				Namespace: "default",
+			},
+		},
+	}
+	customHdfsSecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "hdfs-custom-secret",
+			Namespace: "default",
+		},
+		Data: map[string][]byte{
+			hdfs.HdfsNamenode:      []byte("https://testdomain:port"),
+			hdfs.HdfsRootPath:      []byte("/"),
+			hdfs.KerberosPrincipal: []byte("account@REALM"),
+			hdfs.KerberosKeytab:    []byte("AAA="),
+		},
+	}
+
+	scenarios := map[string]struct {
+		serviceAccount        *v1.ServiceAccount
+		inputConfiguration    *knservingv1.Configuration
+		expectedConfiguration *knservingv1.Configuration
+		shouldFail            bool
+	}{
+		"Custom HDFS Secret": {
+			serviceAccount: customOnlyServiceAccount,
+			inputConfiguration: &knservingv1.Configuration{
+				Spec: knservingv1.ConfigurationSpec{
+					Template: knservingv1.RevisionTemplateSpec{
+						Spec: knservingv1.RevisionSpec{
+							PodSpec: v1.PodSpec{
+								Containers: []v1.Container{
+									{},
+								},
+							},
+						},
+					},
+				},
+			},
+			expectedConfiguration: &knservingv1.Configuration{
+				Spec: knservingv1.ConfigurationSpec{
+					Template: knservingv1.RevisionTemplateSpec{
+						Spec: knservingv1.RevisionSpec{
+							PodSpec: v1.PodSpec{
+								Containers: []v1.Container{
+									{
+										VolumeMounts: []v1.VolumeMount{
+											{
+												Name:      hdfs.HdfsVolumeName,
+												ReadOnly:  true,
+												MountPath: hdfs.MountPath,
+											},
+										},
+									},
+								},
+								Volumes: []v1.Volume{
+									{
+										Name: hdfs.HdfsVolumeName,
+										VolumeSource: v1.VolumeSource{
+											Secret: &v1.SecretVolumeSource{
+												SecretName: "hdfs-custom-secret",
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			shouldFail: false,
+		},
+	}
+
+	g.Expect(c.Create(context.TODO(), customHdfsSecret)).NotTo(gomega.HaveOccurred())
+	g.Expect(c.Create(context.TODO(), customOnlyServiceAccount)).NotTo(gomega.HaveOccurred())
+
+	builder := NewCredentialBulder(c, configMap)
+	for name, scenario := range scenarios {
+
+		err := builder.CreateSecretVolumeAndEnv(scenario.serviceAccount.Namespace, scenario.serviceAccount.Name,
+			&scenario.inputConfiguration.Spec.Template.Spec.Containers[0],
+			&scenario.inputConfiguration.Spec.Template.Spec.Volumes,
+		)
+		if scenario.shouldFail && err == nil {
+			t.Errorf("Test %q failed: returned success but expected error", name)
+		}
+		// Validate
+		if !scenario.shouldFail {
+			if err != nil {
+				t.Errorf("Test %q failed: returned error: %v", name, err)
+			}
+			if diff := cmp.Diff(scenario.expectedConfiguration, scenario.inputConfiguration); diff != "" {
+				t.Errorf("Test %q unexpected configuration spec (-want +got): %v", name, diff)
+			}
+		}
+	}
+
+	g.Expect(c.Delete(context.TODO(), customHdfsSecret)).NotTo(gomega.HaveOccurred())
+	g.Expect(c.Delete(context.TODO(), customOnlyServiceAccount)).NotTo(gomega.HaveOccurred())
+}
+
 func TestAzureStorageAccessKeyCredentialBuilder(t *testing.T) {
 	g := gomega.NewGomegaWithT(t)
 	customOnlyServiceAccount := &v1.ServiceAccount{
@@ -523,4 +634,4 @@ func TestAzureStorageAccessKeyCredentialBuilder(t *testing.T) {
 
 	g.Expect(c.Delete(context.TODO(), customAzureSecret)).NotTo(gomega.HaveOccurred())
 	g.Expect(c.Delete(context.TODO(), customOnlyServiceAccount)).NotTo(gomega.HaveOccurred())
-}
+}
\ No newline at end of file
diff --git a/pkg/webhook/admission/pod/storage_initializer_injector_test.go b/pkg/webhook/admission/pod/storage_initializer_injector_test.go
--- a/pkg/webhook/admission/pod/storage_initializer_injector_test.go
+++ b/pkg/webhook/admission/pod/storage_initializer_injector_test.go
@@ -276,7 +276,7 @@ func TestStorageInitializerInjector(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{
 					Namespace: "default",
 					Annotations: map[string]string{
-						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://<bucket-placeholder>/foo/bar",
+						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://foo/bar",
 						constants.StorageSpecAnnotationKey:                         "true",
 						constants.StorageSpecParamAnnotationKey:                    `{"type": "s3", "bucket": "my-bucket"}`,
 					},
@@ -292,7 +292,7 @@ func TestStorageInitializerInjector(t *testing.T) {
 			expected: &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Annotations: map[string]string{
-						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://<bucket-placeholder>/foo/bar",
+						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://foo/bar",
 						constants.StorageSpecAnnotationKey:                         "true",
 						constants.StorageSpecParamAnnotationKey:                    `{"type": "s3", "bucket": "my-bucket"}`,
 					},
@@ -314,7 +314,7 @@ func TestStorageInitializerInjector(t *testing.T) {
 						{
 							Name:  "storage-initializer",
 							Image: StorageInitializerContainerImage + ":" + StorageInitializerContainerImageVersion,
-							Args:  []string{"s3://<bucket-placeholder>/foo/bar", constants.DefaultModelLocalMountPath},
+							Args:  []string{"s3://my-bucket/foo/bar", constants.DefaultModelLocalMountPath},
 							Env: []v1.EnvVar{
 								{
 									Name:  credentials.StorageOverrideConfigEnvKey,
@@ -739,14 +739,14 @@ func TestCredentialInjection(t *testing.T) {
 					Namespace: "default",
 				},
 				StringData: map[string]string{
-					"my-storage": `{"type": "s3", "bucket": "my-bucket"}`,
+					"my-storage": `{"type": "s3", "bucket": "my-bucket", "region": "na"}`,
 				},
 			},
 			original: &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Namespace: "default",
 					Annotations: map[string]string{
-						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://<bucket-placeholder>/foo/bar",
+						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://foo/bar",
 						constants.StorageSpecAnnotationKey:                         "true",
 						constants.StorageSpecParamAnnotationKey:                    `{"some-param": "some-val"}`,
 						constants.StorageSpecKeyAnnotationKey:                      "my-storage",
@@ -763,7 +763,7 @@ func TestCredentialInjection(t *testing.T) {
 			expected: &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Annotations: map[string]string{
-						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://<bucket-placeholder>/foo/bar",
+						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://foo/bar",
 						constants.StorageSpecAnnotationKey:                         "true",
 						constants.StorageSpecParamAnnotationKey:                    `{"some-param":"some-val"}`,
 						constants.StorageSpecKeyAnnotationKey:                      "my-storage",
@@ -786,7 +786,7 @@ func TestCredentialInjection(t *testing.T) {
 						{
 							Name:  "storage-initializer",
 							Image: StorageInitializerContainerImage + ":" + StorageInitializerContainerImageVersion,
-							Args:  []string{"s3://<bucket-placeholder>/foo/bar", constants.DefaultModelLocalMountPath},
+							Args:  []string{"s3://my-bucket/foo/bar", constants.DefaultModelLocalMountPath},
 							Env: []v1.EnvVar{
 								{
 									Name: credentials.StorageConfigEnvKey,
@@ -836,14 +836,14 @@ func TestCredentialInjection(t *testing.T) {
 					Namespace: "default",
 				},
 				StringData: map[string]string{
-					credentials.DefaultStorageSecretKey: `{"type": "s3", "bucket": "my-bucket"}`,
+					credentials.DefaultStorageSecretKey: `{"type": "s3", "bucket": "my-bucket", "region": "na"}`,
 				},
 			},
 			original: &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Namespace: "default",
 					Annotations: map[string]string{
-						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://<bucket-placeholder>/foo/bar",
+						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://foo/bar",
 						constants.StorageSpecAnnotationKey:                         "true",
 						constants.StorageSpecParamAnnotationKey:                    `{"some-param": "some-val"}`,
 					},
@@ -859,7 +859,7 @@ func TestCredentialInjection(t *testing.T) {
 			expected: &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Annotations: map[string]string{
-						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://<bucket-placeholder>/foo/bar",
+						constants.StorageInitializerSourceUriInternalAnnotationKey: "<scheme-placeholder>://foo/bar",
 						constants.StorageSpecAnnotationKey:                         "true",
 						constants.StorageSpecParamAnnotationKey:                    `{"some-param":"some-val"}`,
 					},
@@ -881,7 +881,7 @@ func TestCredentialInjection(t *testing.T) {
 						{
 							Name:  "storage-initializer",
 							Image: StorageInitializerContainerImage + ":" + StorageInitializerContainerImageVersion,
-							Args:  []string{"s3://<bucket-placeholder>/foo/bar", constants.DefaultModelLocalMountPath},
+							Args:  []string{"s3://my-bucket/foo/bar", constants.DefaultModelLocalMountPath},
 							Env: []v1.EnvVar{
 								{
 									Name: credentials.StorageConfigEnvKey,
diff --git a/python/kserve/test/test_storage.py b/python/kserve/test/test_storage.py
--- a/python/kserve/test/test_storage.py
+++ b/python/kserve/test/test_storage.py
@@ -197,10 +197,3 @@ def test_unpack_zip_file():
     kserve.Storage._unpack_archive_file(tar_file, mimetype, out_dir)
     assert os.path.exists(os.path.join(out_dir, 'model.pth'))
     os.remove(os.path.join(out_dir, 'model.pth'))
-
-
-def test_storage_spec_update():
-    uri = 's3://<bucket-placeholder>/bar'
-    expected_uri = 's3://default/bar'
-    os.environ["STORAGE_CONFIG"] = '{"type": "s3", "bucket": "default"}'
-    assert expected_uri == kserve.Storage._update_with_storage_spec(uri)