Repository: kserve/kserve

Issue:
SDK for patching kfservice is returning 415 errors
/kind bug

**What steps did you take and what happened:**
[A clear and concise description of what the bug is.]
I try to follow this [example notebook](https://github.com/kubeflow/kfserving/blob/master/sdk/sample/kfserving_sdk_sample.ipynb) to use the kfserving SDK. However, during the 5th python cell at `KFServing.patch('flower-sample', kfsvc)`, it returns the following errors
```
---------------------------------------------------------------------------
ApiException                              Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/kfserving/api/kf_serving_client.py in patch(self, name, kfservice, namespace)
     79                 name,
---> 80                 kfservice)
     81         except client.rest.ApiException as e:

/opt/conda/lib/python3.7/site-packages/kubernetes/client/apis/custom_objects_api.py in patch_namespaced_custom_object(self, group, version, namespace, plural, name, body, **kwargs)
   1950         else:
-> 1951             (data) = self.patch_namespaced_custom_object_with_http_info(group, version, namespace, plural, name, body, **kwargs)
   1952             return data

/opt/conda/lib/python3.7/site-packages/kubernetes/client/apis/custom_objects_api.py in patch_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, name, body, **kwargs)
   2056                                         _request_timeout=params.get('_request_timeout'),
-> 2057                                         collection_formats=collection_formats)
   2058 

/opt/conda/lib/python3.7/site-packages/kubernetes/client/api_client.py in call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, async_req, _return_http_data_only, collection_formats, _preload_content, _request_timeout)
    333                                    response_type, auth_settings,
--> 334                                    _return_http_data_only, collection_formats, _preload_content, _request_timeout)
    335         else:

/opt/conda/lib/python3.7/site-packages/kubernetes/client/api_client.py in __call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout)
    167                                      _preload_content=_preload_content,
--> 168                                      _request_timeout=_request_timeout)
    169 

/opt/conda/lib/python3.7/site-packages/kubernetes/client/api_client.py in request(self, method, url, query_params, headers, post_params, body, _preload_content, _request_timeout)
    392                                           _request_timeout=_request_timeout,
--> 393                                           body=body)
    394         elif method == "DELETE":

/opt/conda/lib/python3.7/site-packages/kubernetes/client/rest.py in PATCH(self, url, headers, query_params, post_params, body, _preload_content, _request_timeout)
    285                             _request_timeout=_request_timeout,
--> 286                             body=body)
    287 

/opt/conda/lib/python3.7/site-packages/kubernetes/client/rest.py in request(self, method, url, query_params, headers, body, post_params, _preload_content, _request_timeout)
    221         if not 200 <= r.status <= 299:
--> 222             raise ApiException(http_resp=r)
    223 

ApiException: (415)
Reason: Unsupported Media Type
HTTP response headers: HTTPHeaderDict({'Content-Type': 'application/json', 'Date': 'Wed, 07 Aug 2019 21:05:59 GMT', 'Content-Length': '263'})
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the body of the request was in an unknown format - accepted media types include: application/json-patch+json, application/merge-patch+json","reason":"UnsupportedMediaType","code":415}



During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
/usr/local/bin/kernel-launchers/python/scripts/launch_ipykernel.py in <module>
----> 1 KFServing.patch('flower-sample', kfsvc)

/opt/conda/lib/python3.7/site-packages/kfserving/api/kf_serving_client.py in patch(self, name, kfservice, namespace)
     82             raise RuntimeError(
     83                 "Exception when calling CustomObjectsApi->patch_namespaced_custom_object:\
---> 84                  %s\n" % e)
     85 
     86     def delete(self, name, namespace=None):

RuntimeError: Exception when calling CustomObjectsApi->patch_namespaced_custom_object:                 (415)
Reason: Unsupported Media Type
HTTP response headers: HTTPHeaderDict({'Content-Type': 'application/json', 'Date': 'Wed, 07 Aug 2019 21:05:59 GMT', 'Content-Length': '263'})
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the body of the request was in an unknown format - accepted media types include: application/json-patch+json, application/merge-patch+json","reason":"UnsupportedMediaType","code":415}
```

**What did you expect to happen:**
The KFServing.patch should update the new changes instead of submitting the wrong media type


**Anything else you would like to add:**
[Miscellaneous information that will assist in solving the issue.]


**Environment:**

- Istio Version: 1.1.5
- Knative Version: 0.6
- KFServing Version: 0.1
- Kubeflow version: 0.5.1
- Minikube version: 
- Kubernetes version: (use `kubectl version`): 1.13.8
- OS (e.g. from `/etc/os-release`): ubuntu 16

KFServing SDK should have an easy way to get KFService status
/kind feature

**Describe the solution you'd like**
We need a better way SDK to show the status whether KFService is ready or not, rather than manually looking for in the dump

```
{'apiVersion': 'serving.kubeflow.org/v1alpha1',
'kind': 'KFService',
'metadata': {'creationTimestamp': '2019-08-13T05:19:35Z',
 'generation': 2,
 'name': 'flower-sample',
 'namespace': 'kubeflow',
 'resourceVersion': '17220979',
 'selfLink': '/apis/serving.kubeflow.org/v1alpha1/namespaces/kubeflow/kfservices/flower-sample',
 'uid': 'f0b2bc89-bd89-11e9-a6b0-6a897a432cf6'},
'spec': {'default': {'tensorflow': {'modelUri': 'gs://kfserving-samples/models/tensorflow/flowers',
   'resources': {'requests': {'cpu': '1', 'memory': '2Gi'}},
   'runtimeVersion': 'latest'}}},
'status': {'canary': {},
 'conditions': [{'lastTransitionTime': '2019-08-13T05:19:38Z',
   'status': 'Unknown',
   'type': 'DefaultPredictorReady'},
  {'lastTransitionTime': '2019-08-13T05:19:38Z',
   'message': 'Configuration "flower-sample-default" is waiting for a Revision to become ready.',
   'reason': 'RevisionMissing',
   'status': 'Unknown',
   'type': 'Ready'},
  {'lastTransitionTime': '2019-08-13T05:19:38Z',
   'message': 'Configuration "flower-sample-default" is waiting for a Revision to become ready.',
   'reason': 'RevisionMissing',
   'status': 'Unknown',
   'type': 'RoutesReady'}],
 'default': {'name': 'flower-sample-default-g5jz2'},
 'url': 'flower-sample.kubeflow.example.com'}}
```


Discussion:
/assign @jinchihe 
So I discussed with @jinchihe the issue is happening only on pypi kubernetes sdk 10.0.0. Jin Chi will look into it more on the new changes in 10.0.

Just has basic checked with @Tomcli , That works fine with kubernetes Client 9.0.0.
```
(python36) [root@jinchi1 ~]# pip list |grep -i kube
kubernetes                    9.0.0
```
The problem only exists for kubernetes client SDK 10.0.0, we need to check what's changed in the new kubernetes client SDK, and handle that. 

I'm going to enhance that. Thanks.
Thanks folks 
@Tomcli @animeshsingh 

This a bug of kubernetes python SDK client in the new verion (10.0), please see here: https://github.com/kubernetes-client/python/issues/866

The problem has been fixed in the  kubernetes python SDK. Here I think we can pin the `kubernetes=9.0.0`, and after next kubernetes sdk release, we can pin to `>=next version`. 
Agree?
Thanks @jinchihe - works!
/assign @jinchihe 
cc @ellis-bigelow 
@animeshsingh Thanks, totally agree with you. I think we should have a API to `watching` the kfservice status. For now, the `KFServingClient.get` API need to user polling manually.
/cc @yuzisun 
@jinchihe any update on this?
> @jinchihe any update on this?

Sorry for later, Lots of tasks from other tracks. I'm going to start this now, thanks.

Resolving patch:
diff --git a/python/kfserving/kfserving/api/kf_serving_client.py b/python/kfserving/kfserving/api/kf_serving_client.py
--- a/python/kfserving/kfserving/api/kf_serving_client.py
+++ b/python/kfserving/kfserving/api/kf_serving_client.py
@@ -17,6 +17,7 @@
 from ..constants import constants
 from ..utils import utils
 from .creds_utils import set_gcs_credentials, set_s3_credentials, set_azure_credentials
+from .kf_serving_watch import watch as kfsvc_watch
 
 
 class KFServingClient(object):
@@ -73,14 +74,14 @@ def set_credentials(self, storage_type, namespace=None, credentials_file=None,
                 currently.\n" % storage_type)
 
 
-    def create(self, kfservice, namespace=None):
+    def create(self, kfservice, namespace=None, watch=False, timeout_seconds=600): #pylint:disable=inconsistent-return-statements
         """Create the provided KFService in the specified namespace"""
 
         if namespace is None:
             namespace = utils.set_kfsvc_namespace(kfservice)
 
         try:
-            return self.api_instance.create_namespaced_custom_object(
+            outputs = self.api_instance.create_namespaced_custom_object(
                 constants.KFSERVING_GROUP,
                 constants.KFSERVING_VERSION,
                 namespace,
@@ -91,31 +92,65 @@ def create(self, kfservice, namespace=None):
                 "Exception when calling CustomObjectsApi->create_namespaced_custom_object:\
                  %s\n" % e)
 
-    def get(self, name, namespace=None):
+        if watch:
+            kfsvc_watch(
+                name=outputs['metadata']['name'],
+                namespace=namespace,
+                timeout_seconds=timeout_seconds)
+        else:
+            return outputs
+
+
+    def get(self, name=None, namespace=None, watch=False, timeout_seconds=600): #pylint:disable=inconsistent-return-statements
         """Get the created KFService in the specified namespace"""
 
         if namespace is None:
             namespace = utils.get_default_target_namespace()
 
-        try:
-            return self.api_instance.get_namespaced_custom_object(
-                constants.KFSERVING_GROUP,
-                constants.KFSERVING_VERSION,
-                namespace,
-                constants.KFSERVING_PLURAL,
-                name)
-        except client.rest.ApiException as e:
-            raise RuntimeError(
-                "Exception when calling CustomObjectsApi->get_namespaced_custom_object: %s\n" % e)
-
-    def patch(self, name, kfservice, namespace=None):
+        if name:
+            if watch:
+                kfsvc_watch(
+                    name=name,
+                    namespace=namespace,
+                    timeout_seconds=timeout_seconds)
+            else:
+                try:
+                    return self.api_instance.get_namespaced_custom_object(
+                        constants.KFSERVING_GROUP,
+                        constants.KFSERVING_VERSION,
+                        namespace,
+                        constants.KFSERVING_PLURAL,
+                        name)
+                except client.rest.ApiException as e:
+                    raise RuntimeError(
+                        "Exception when calling CustomObjectsApi->get_namespaced_custom_object:\
+                        %s\n" % e)
+        else:
+            if watch:
+                kfsvc_watch(
+                    namespace=namespace,
+                    timeout_seconds=timeout_seconds)
+            else:
+                try:
+                    return self.api_instance.list_namespaced_custom_object(
+                        constants.KFSERVING_GROUP,
+                        constants.KFSERVING_VERSION,
+                        namespace,
+                        constants.KFSERVING_PLURAL)
+                except client.rest.ApiException as e:
+                    raise RuntimeError(
+                        "Exception when calling CustomObjectsApi->list_namespaced_custom_object:\
+                        %s\n" % e)
+
+
+    def patch(self, name, kfservice, namespace=None, watch=False, timeout_seconds=600): # pylint:disable=too-many-arguments,inconsistent-return-statements
         """Patch the created KFService in the specified namespace"""
 
         if namespace is None:
             namespace = utils.set_kfsvc_namespace(kfservice)
 
         try:
-            return self.api_instance.patch_namespaced_custom_object(
+            outputs = self.api_instance.patch_namespaced_custom_object(
                 constants.KFSERVING_GROUP,
                 constants.KFSERVING_VERSION,
                 namespace,
@@ -127,6 +162,14 @@ def patch(self, name, kfservice, namespace=None):
                 "Exception when calling CustomObjectsApi->patch_namespaced_custom_object:\
                  %s\n" % e)
 
+        if watch:
+            kfsvc_watch(
+                name=outputs['metadata']['name'],
+                namespace=namespace,
+                timeout_seconds=timeout_seconds)
+        else:
+            return outputs
+
     def delete(self, name, namespace=None):
         """Delete the provided KFService in the specified namespace"""
 
diff --git a/python/kfserving/kfserving/api/kf_serving_watch.py b/python/kfserving/kfserving/api/kf_serving_watch.py
new file mode 100644
--- /dev/null
+++ b/python/kfserving/kfserving/api/kf_serving_watch.py
@@ -0,0 +1,58 @@
+# Copyright 2019 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from kubernetes import client
+from kubernetes import watch as k8s_watch
+from table_logger import TableLogger
+
+from ..constants import constants
+from ..utils import utils
+
+
+def watch(name=None, namespace=None, timeout_seconds=600):
+    """Watch the created or patched KFService in the specified namespace"""
+
+    if namespace is None:
+        namespace = utils.get_default_target_namespace()
+
+    tbl = TableLogger(
+        columns='NAME,READY,DEFAULT_TRAFFIC,CANARY_TRAFFIC,URL',
+        colwidth={'NAME': 20, 'READY':10, 'DEFAULT_TRAFFIC':15, 'CANARY_TRAFFIC':15, 'URL': 50},
+        border=False)
+
+    stream = k8s_watch.Watch().stream(
+        client.CustomObjectsApi().list_namespaced_custom_object,
+        constants.KFSERVING_GROUP,
+        constants.KFSERVING_VERSION,
+        namespace,
+        constants.KFSERVING_PLURAL,
+        timeout_seconds=timeout_seconds)
+
+    for event in stream:
+        kfserivce = event['object']
+        kfsvc_name = kfserivce['metadata']['name']
+        if name and name != kfsvc_name:
+            continue
+        else:
+            url = kfserivce['status'].get('url', '')
+            default_traffic = kfserivce['status'].get('default', {}).get('traffic', '')
+            canary_traffic = kfserivce['status'].get('canary', {}).get('traffic', '')
+            status = 'Unknown'
+            for condition in kfserivce['status'].get('conditions', {}):
+                if condition.get('type', '') == 'Ready':
+                    status = condition.get('status', 'Unknown')
+            tbl(kfsvc_name, status, default_traffic, canary_traffic, url)
+
+            if name == kfsvc_name and status == 'True':
+                break


Test patch:
diff --git a/python/kfserving/test/test_kfservice_client.py b/python/kfserving/test/test_kfservice_client.py
--- a/python/kfserving/test/test_kfservice_client.py
+++ b/python/kfserving/test/test_kfservice_client.py
@@ -39,28 +39,35 @@ def generate_kfservice():
                               spec=V1alpha2KFServiceSpec(default=default_model_spec))
     return kfsvc
 
-# Unit test for kfserving create api
 def test_kfservice_client_creat():
+    '''Unit test for kfserving create api'''
     with patch('kfserving.api.kf_serving_client.KFServingClient.create',
                return_value=mocked_unit_result):
         kfsvc = generate_kfservice()
         assert mocked_unit_result == KFServing.create(kfsvc, namespace='kubeflow')
 
-# Unit test for kfserving get api
 def test_kfservice_client_get():
+    '''Unit test for kfserving get api'''
     with patch('kfserving.api.kf_serving_client.KFServingClient.get',
                return_value=mocked_unit_result):
         assert mocked_unit_result == KFServing.get('flower-sample', namespace='kubeflow')
 
-# Unit test for kfserving patch api
+def test_kfservice_client_watch():
+    '''Unit test for kfserving get api'''
+    with patch('kfserving.api.kf_serving_client.KFServingClient.get',
+               return_value=mocked_unit_result):
+        assert mocked_unit_result == KFServing.get('flower-sample', namespace='kubeflow',
+                                                   watch=True, timeout_seconds=120)
+
 def test_kfservice_client_patch():
+    '''Unit test for kfserving patch api'''
     with patch('kfserving.api.kf_serving_client.KFServingClient.patch',
                return_value=mocked_unit_result):
         kfsvc = generate_kfservice()
         assert mocked_unit_result == KFServing.patch('flower-sample', kfsvc, namespace='kubeflow')
 
-# Unit test for kfserving delete api
 def test_kfservice_client_delete():
+    '''Unit test for kfserving delete api'''
     with patch('kfserving.api.kf_serving_client.KFServingClient.delete',
                return_value=mocked_unit_result):
         assert mocked_unit_result == KFServing.delete('flower-sample', namespace='kubeflow')