Repository: kserve/kserve

Issue:
Azure anonymous access fails when secret is configured
/kind bug

**What steps did you take and what happened:**
Create the azure azcreds secret
Attempt to use a public blob
Access fails since the SP does not have access to the blob

**What did you expect to happen:**
KFServing should attempt anonymous access before using the secret

**Anything else you would like to add:**
[Miscellaneous information that will assist in solving the issue.]


**Environment:**

- Istio Version:
- Knative Version:
- KFServing Version:
- Kubeflow version:
- Minikube version:
- Kubernetes version: (use `kubectl version`):
- OS (e.g. from `/etc/os-release`):



Discussion:

Resolving patch:
diff --git a/python/kfserving/kfserving/storage.py b/python/kfserving/kfserving/storage.py
--- a/python/kfserving/kfserving/storage.py
+++ b/python/kfserving/kfserving/storage.py
@@ -17,6 +17,7 @@
 import tempfile
 import os
 import re
+from azure.common import AzureMissingResourceHttpError
 from azure.storage.blob import BlockBlobService
 from google.auth import exceptions
 from google.cloud import storage
@@ -111,9 +112,15 @@ def _download_blob(uri, out_dir: str): # pylint: disable=too-many-locals
                      account_name,
                      container_name,
                      prefix)
-        token = Storage._get_azure_storage_token()
-        block_blob_service = BlockBlobService(account_name=account_name, token_credential=token)
-        blobs = block_blob_service.list_blobs(container_name, prefix=prefix)
+        try:
+            block_blob_service = BlockBlobService(account_name=account_name)
+            blobs = block_blob_service.list_blobs(container_name, prefix=prefix)
+        except AzureMissingResourceHttpError:
+            token = Storage._get_azure_storage_token()
+            if token is None:
+                logging.warning("Azure credentials not found, retrying anonymous access")
+            block_blob_service = BlockBlobService(account_name=account_name, token_credential=token)
+            blobs = block_blob_service.list_blobs(container_name, prefix=prefix)
 
         for blob in blobs:
             dest_path = os.path.join(out_dir, blob.name)
@@ -156,6 +163,8 @@ def _get_azure_storage_token():
 
         token_credential = TokenCredential(token["accessToken"])
 
+        logging.info("Retrieved SP token credential for client_id: %s", client_id)
+
         return token_credential
 
     @staticmethod


Test patch:
diff --git a/python/kfserving/test/test_azure_storage.py b/python/kfserving/test/test_azure_storage.py
--- a/python/kfserving/test/test_azure_storage.py
+++ b/python/kfserving/test/test_azure_storage.py
@@ -14,6 +14,8 @@
 
 import unittest.mock as mock
 import itertools
+import pytest
+from azure.common import AzureMissingResourceHttpError
 import kfserving
 
 def create_mock_item(path):
@@ -37,9 +39,8 @@ def get_call_args(call_args_list):
 # pylint: disable=protected-access
 
 @mock.patch('kfserving.storage.os.makedirs')
-@mock.patch('kfserving.storage.Storage._get_azure_storage_token')
 @mock.patch('kfserving.storage.BlockBlobService')
-def test_blob(mock_storage, mock_get_token, mock_makedirs): # pylint: disable=unused-argument
+def test_blob(mock_storage, mock_makedirs): # pylint: disable=unused-argument
 
     # given
     blob_path = 'https://kfserving.blob.core.windows.net/tensorrt/simple_string/'
@@ -56,10 +57,38 @@ def test_blob(mock_storage, mock_get_token, mock_makedirs): # pylint: disable=un
         ('tensorrt', 'simple_string/config.pbtxt', 'dest_path/config.pbtxt')
         ]
 
+    mock_storage.assert_called_with(account_name="kfserving")
+
 @mock.patch('kfserving.storage.os.makedirs')
 @mock.patch('kfserving.storage.Storage._get_azure_storage_token')
 @mock.patch('kfserving.storage.BlockBlobService')
-def test_deep_blob(mock_storage, mock_get_token, mock_makedirs): # pylint: disable=unused-argument
+def test_secure_blob(mock_storage, mock_get_token, mock_makedirs): # pylint: disable=unused-argument
+
+    # given
+    blob_path = 'https://kfsecured.blob.core.windows.net/tensorrt/simple_string/'
+    mock_blob = mock_storage.return_value
+    mock_blob.list_blobs.side_effect = AzureMissingResourceHttpError("fail auth", 404)
+    mock_get_token.return_value = "some_token"
+
+    # when
+    with pytest.raises(AzureMissingResourceHttpError):
+        kfserving.Storage._download_blob(blob_path, "dest_path")
+
+    # then
+    mock_get_token.assert_called()
+    arg_list = []
+    for call in mock_storage.call_args_list:
+        _, kwargs = call
+        arg_list.append(kwargs)
+
+    assert arg_list == [
+        {"account_name": "kfsecured"},
+        {"account_name": "kfsecured", "token_credential": "some_token"},
+        ]
+
+@mock.patch('kfserving.storage.os.makedirs')
+@mock.patch('kfserving.storage.BlockBlobService')
+def test_deep_blob(mock_storage, mock_makedirs): # pylint: disable=unused-argument
 
     # given
     blob_path = 'https://accountname.blob.core.windows.net/container/some/deep/blob/path'
@@ -77,9 +106,8 @@ def test_deep_blob(mock_storage, mock_get_token, mock_makedirs): # pylint: disab
     assert actual_calls == expected_calls
 
 @mock.patch('kfserving.storage.os.makedirs')
-@mock.patch('kfserving.storage.Storage._get_azure_storage_token')
 @mock.patch('kfserving.storage.BlockBlobService')
-def test_blob_file(mock_storage, mock_get_token, mock_makedirs): # pylint: disable=unused-argument
+def test_blob_file(mock_storage, mock_makedirs): # pylint: disable=unused-argument
 
     # given
     blob_path = 'https://accountname.blob.core.windows.net/container/somefile'
@@ -97,9 +125,8 @@ def test_blob_file(mock_storage, mock_get_token, mock_makedirs): # pylint: disab
     assert actual_calls == expected_calls
 
 @mock.patch('kfserving.storage.os.makedirs')
-@mock.patch('kfserving.storage.Storage._get_azure_storage_token')
 @mock.patch('kfserving.storage.BlockBlobService')
-def test_blob_fq_file(mock_storage, mock_get_token, mock_makedirs): # pylint: disable=unused-argument
+def test_blob_fq_file(mock_storage, mock_makedirs): # pylint: disable=unused-argument
 
     # given
     blob_path = 'https://accountname.blob.core.windows.net/container/folder/somefile'
@@ -117,9 +144,8 @@ def test_blob_fq_file(mock_storage, mock_get_token, mock_makedirs): # pylint: di
     assert actual_calls == expected_calls
 
 @mock.patch('kfserving.storage.os.makedirs')
-@mock.patch('kfserving.storage.Storage._get_azure_storage_token')
 @mock.patch('kfserving.storage.BlockBlobService')
-def test_blob_no_prefix(mock_storage, mock_get_token, mock_makedirs): # pylint: disable=unused-argument
+def test_blob_no_prefix(mock_storage, mock_makedirs): # pylint: disable=unused-argument
 
     # given
     blob_path = 'https://accountname.blob.core.windows.net/container/'