Repository: kserve/kserve

Issue:
Transformer not working with Tensorflow tensors in column format
/kind bug

**What steps did you take and what happened:**

Following this guide: https://github.com/kubeflow/kfserving/tree/master/docs/samples/v1beta1/transformer/torchserve_image_transformer

I created a transformer converting base64 to data with tensors in column format to be sent to a predictor running tensorflow  serving 2.5.1.

Here's a part of the final data I send - the result of `preprocess` of `ImageTransformer(kfserving.KFModel)`
```json
{
   "inputs": {
      "input_image": [
         [
              ...data
         ]
    }
}
```

But I get an error because `inputs` is not a list. I believe it is from this line here https://github.com/kubeflow/kfserving/blob/master/python/kfserving/kfserving/handlers/http.py#L45

However, tensorflow serving does not allow it being a list, but expects a dict/map (as far as I understand) https://www.tensorflow.org/tfx/serving/api_rest#specifying_input_tensors_in_column_format

**What did you expect to happen:**

Not to get an error that inputs has to be a list.

**Anything else you would like to add:**

The request succeeds when I remove the validation check. 

I'm pretty new to this, so I might be misunderstanding something. But I think the data should be allowed to be a dict for tensorflow serving. 


Discussion:
@Vafilor According to inference [v2 protocol](https://github.com/kubeflow/kfserving/blob/master/docs/predict-api/v2/required_api.md#inference-request-json-object) the `inputs` is a list but looks like this clashes with the tensorflow v1 protocol where `inputs` is a dict. We need a better way to validate the request but can remove the `inputs` check for now.
I'd propose to validate according to the protocol implied by the endpoint.  I.m.o the http handlers initializers and the `KFModel.__call__` need to receive the protocol version so that each specific server can further validate or map the two protocol versions.

Resolving patch:
diff --git a/python/custom_transformer/model.py b/python/custom_transformer/model.py
--- a/python/custom_transformer/model.py
+++ b/python/custom_transformer/model.py
@@ -12,18 +12,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import kserve
+from kserve import Model, ModelServer, model_server
+from kserve.model import PredictorProtocol
 from torchvision import transforms
 from typing import Dict
 from PIL import Image
 import base64
 import io
 import argparse
+import numpy
 from tritonclient.grpc.service_pb2 import ModelInferRequest, ModelInferResponse
 from tritonclient.grpc import InferResult, InferInput
 
 
-class ImageTransformer(kserve.Model):
+def image_transform(instance):
+    """converts the input image of Bytes Array into Tensor
+    Args:
+        request input instance: The request input instance for image.
+    Returns:
+        List: Returns the data key's value and converts that into a list
+        after converting it into a tensor
+    """
+    image_processing = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.1307,), (0.3081,))
+    ])
+    byte_array = base64.b64decode(instance["image"]["b64"])
+    image = Image.open(io.BytesIO(byte_array))
+    tensor = image_processing(image).numpy()
+    print(tensor.shape)
+    return tensor
+
+
+class ImageTransformer(Model):
     def __init__(self, name: str, predictor_host: str, protocol: str):
         super().__init__(name)
         self.predictor_host = predictor_host
@@ -31,46 +52,42 @@ def __init__(self, name: str, predictor_host: str, protocol: str):
         self.ready = True
 
     def preprocess(self, request: Dict) -> ModelInferRequest:
-        instances = request["instances"]
         # Input follows the Tensorflow V1 HTTP API for binary values
         # https://www.tensorflow.org/tfx/serving/api_rest#encoding_binary_values
-        data = instances[0]["image"]["b64"]
+        input_tensors = [image_transform(instance) for instance in request["instances"]]
 
-        raw_img_data = base64.b64decode(data)
-        input_image = Image.open(io.BytesIO(raw_img_data))
-        preprocess = transforms.Compose([
-            transforms.ToTensor(),
-            transforms.Normalize((0.1307,), (0.3081,))
-        ])
+        # Transform to KServe v1/v2 inference protocol
+        if self.protocol == PredictorProtocol.GRPC_V2.value:
+            return self.v2_request_transform(numpy.asarray(input_tensors))
+        else:
+            inputs = [{"data": input_tensor.tolist()} for input_tensor in input_tensors]
+            request = {"instances": inputs}
+            return request
 
-        input_tensor = preprocess(input_image).numpy()
-        sp = [1]
-        for p in input_tensor.shape:
-            sp.append(p)
-        input_0 = InferInput(
-            "INPUT__0", sp, "FP32"
-        )
-        input_0.set_data_from_numpy(input_tensor.reshape(sp))
-        inputs = [input_0]
+    def v2_request_transform(self, input_tensors):
         request = ModelInferRequest()
         request.model_name = self.name
-        for infer_input in inputs:
-            request.inputs.extend([infer_input._get_tensor()])
-            if infer_input._get_content() is not None:
-                request.raw_input_contents.extend([infer_input._get_content()])
+        input_0 = InferInput("INPUT__0", input_tensors.shape, "FP32")
+        input_0.set_data_from_numpy(input_tensors)
+        request.inputs.extend([input_0._get_tensor()])
+        if input_0._get_content() is not None:
+            request.raw_input_contents.extend([input_0._get_content()])
         return request
 
     def postprocess(self, infer_response: ModelInferResponse) -> Dict:
-        response = InferResult(infer_response)
-        return {"predictions": response.as_numpy("OUTPUT__0").tolist()}
+        if self.protocol == PredictorProtocol.GRPC_V2.value:
+            response = InferResult(infer_response)
+            return {"predictions": response.as_numpy("OUTPUT__0").tolist()}
+        else:
+            return infer_response
 
 
-parser = argparse.ArgumentParser(parents=[kserve.model_server.parser])
+parser = argparse.ArgumentParser(parents=[model_server.parser])
 parser.add_argument(
     "--predictor_host", help="The URL for the model predict function", required=True
 )
 parser.add_argument(
-    "--protocol", help="The protocol for the predictor", required=True
+    "--protocol", help="The protocol for the predictor", default="v1"
 )
 parser.add_argument(
     "--model_name", help="The name that the model is served under."
@@ -80,4 +97,4 @@ def postprocess(self, infer_response: ModelInferResponse) -> Dict:
 if __name__ == "__main__":
     model = ImageTransformer(args.model_name, predictor_host=args.predictor_host,
                              protocol=args.protocol)
-    kserve.ModelServer(workers=1).start([model])
+    ModelServer(workers=1).start([model])
diff --git a/python/kserve/kserve/model.py b/python/kserve/kserve/model.py
--- a/python/kserve/kserve/model.py
+++ b/python/kserve/kserve/model.py
@@ -106,14 +106,18 @@ def _grpc_client(self):
             self._grpc_client_stub = service_pb2_grpc.GRPCInferenceServiceStub(_channel)
         return self._grpc_client_stub
 
-    @staticmethod
-    def validate(request):
-        if isinstance(request, dict):
-            if ("instances" in request and not isinstance(request["instances"], list)) or \
-               ("inputs" in request and not isinstance(request["inputs"], list)):
+    def validate(self, request):
+        if self.protocol == PredictorProtocol.REST_V2:
+            if "inputs" in request and not isinstance(request["inputs"], list):
                 raise tornado.web.HTTPError(
                     status_code=HTTPStatus.BAD_REQUEST,
-                    reason="Expected \"instances\" or \"inputs\" to be a list"
+                    reason="Expected \"inputs\" to be a list"
+                )
+        elif isinstance(request, Dict) or self.protocol == PredictorProtocol.REST_V1:
+            if "instances" in request and not isinstance(request["instances"], list):
+                raise tornado.web.HTTPError(
+                    status_code=HTTPStatus.BAD_REQUEST,
+                    reason="Expected \"instances\" to be a list"
                 )
         return request
 


Test patch:
diff --git a/test/e2e/data/transformer.json b/test/e2e/data/transformer.json
--- a/test/e2e/data/transformer.json
+++ b/test/e2e/data/transformer.json
@@ -1,8 +1,8 @@
 {
     "instances":[
        {
-          "image_bytes":{
-	     "b64": "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAJhElEQVR4nAXBWW+c13kA4HPes3z7zPfNDGchqaEoyZLqKonj2E5r2GkCJIDj3vSiF73sT+jvCRAjl4GTNAhQB0WRGEjRyI53ubUWmhVFi+QMOfvybWd5T56HvvNv36UOpeAUQKnaWC2ltIgOHQULjDgdUWKFrBjhFJxFow0iUkK5sbRGSglBh5RSpbS1nDoEYhVibkihLFcEnCsJokciIIxzC0CII1RArZRBxh0wRjgQipqYGohFZIr6lnkKmbJA0VI0vgBOAbizWhNqHLGOUMaAOzTE1c4aahlqxQKgBBkjiFYKYZxAzRCtMZY6Bw4ok475pfXGM50rt91q5mziM0mxEQaBZxAUEMoYE4RodJzbmjAHqD1mCKcEABgQRww6AlTIoH/99no5nc4KwSUQTxleuuDR6dR5Lc0iFfvb1fz8ahl73I6Xw55sJ57POXVGUmKd5YRQylNKqXEIYJRRknnWWoeWUCoFfP/HP/n0/gcXy1luuLHR6dnk5PzcSwf7vUPnJYp7It4x1XZ2dRGmrbPtZYXYS0QomNUFOMJrSFZFaE2dxabBLHcOjaKOODTAoCgW7//H7y6X9eUWTs8Xp6PnzI8ta0SNjghj7gceBR+iqSoH+8OqzE9OLueritH4+k4sLFJrYFKycZn+5k+P/3w0qoALwalzjIGUghKkYE9OT87GMyczFu9BthsMrsl2W1FsZFF/JxZmXS4uEolpJFVViqQ7yeH55aaqCaOcoAPePCxorOXOvEgK5VvnrDOIBsDTtjHO4/ONpXEr611P271Op5vEaZK0VK2r7TqL/Fhyq0pn1Go+I2jLPGcyvFqb0aqynAEncOfbr/EgiZs7r/39W2GyqwxFJlBGCrKke+986ovo2t7BvTjeEcLHWpfr3FnCKP/qwZejs7MwiqIwns3mi+WKUsiSwON8sdUn45VmPpWSh832wY3bpSbDw1sd7ZYnp9oZa8LXfvBPwxuvHH7r2aefP8ji/sXVlDvpCUEc2eb5ajHPIuEIseg6Ozu1NtPFijJI4ogzrqri6fOznTR4YT8B5sUXl5OXvvdq1GwzL7TGMeCnzzcyOyThfhJ1fR4HMvSlR9Du7Q6qqpBSrjebZta+fffFRqPZ7fUpMAoszVqcUcYgCFMqW8fPN2dXOQi/UVWqrrWQYRg1Ij9oCB5z+4uf/fyrh0eT6Vh6AGAOb+wFEVhT9rsdzqFW6satWzdv3WZClFW1zgtjsSyrNG16vt9I242sy4LsbDQFykSxzauiFMLb5JawQBAcpGx6fnxxdnx6fnRy9oQKu3fQ3x32pGStND0YDuM4GezuLddrbfFyMkNHKeNFWZVlSQmJ4qjVaWXt1DrkBB1zOOi0Q997/8v/zwy+0BK+ZyWvJlfPsF4Mbx4y3wsbWae3P5tvV+vCWrKzs8OFVymjtCmr2lhrrK1qZQy0O11KhaSVR411IRecNeMgTQKKZu2i6YJ2Eh5JYUE/u3jWy5oHt16sNPno00fno0USZ0L4Xx1/QwgggVqZbV6mrZZxdHR5FSVNzlwYhlJ6RM9svux1E2CU9rt9TgCrerB/OFHhku5uWbfZaTUbQvjJ9VsvvvJ3b56fXxVFcXl1NRqPBSf9TFTz03w5bjai+XRyOR6t1yujTegHzGmh5qy46Ee6HVAupdfI+sZyj3u3D4effJqsxS2km96eePjow9f/4V8/uP9hnq+1ml6NnxMCWw2c6AwWe8F6NfnasKzXzaw1ZVlVZZELz+BWV+ddUe7GYW1KHsVR1ukYyiuQftxI0+Y3z8dvvPq31RbDZDI6Pzs+OjJWASP5epW0B6tV0Yz9O7fvffzg8WePn73xw58KGT49Pl5tCiRQlduDXhJEQauVOG6McoCmaLZi5ovCOgIwvLZfVGpVoIiG126+PLoYPXr0uNNu+9Lb2927fnjTUVHWKKNWY+fad199YzKZ3b//QV6Uy9XWk17TjQ7i2Z0BZv6au1lEK9jMRoHQnFYUK4qm02oTYFfz/HScg9+/e+/btTbakuW6SLPeC4c3D3YHs8l0Nl0IL846u/NNNZ6ttxUyPxnsH97sdoZJkALwGrkRaAh/evx0+MLf+KBQldz3fd9PkjhuNO7evfOH//p9sRqHre7x2dW1/eHhnZc9yW8Mh8v54uGjr9HZ86Val7ay3npZdPv738yK1rXmzPMIqqWxjvs1Kv7F8dXw3mtIcmoMQbfebJbLabv10ttv/eil79x9999/SylrNrO93f24kTKTt/p8cKhXgf/5gwejLXWi0ey3OzebjPvW0ScuOh5byWhZVYUhBhk/WgVTmzhRgVo5ZABsd9B98/WXfWEPD/b+8Z//5de/fW86Xo1WWFXHkph5aY5Px0Rp17mTdUMkjlKBfohUautWVvhC+pzmtNBCONT8aAm/+5//femg05dRKPig3x90Gjdv7BOnRpPZO79877MvHtaVMoYQB84q6zUsCE4CQ5mBwOeEOFopcEA59xmiq4whKBAYBaUpbEH+8bOjX73/0deT7da6k6dfX+tlvhBbxd/9z48/f3hRGM/yBgQp8ROIm+ABYbamUFlrra4NqYxzAIxBGMo0EIEQVEZWhNpRmaS83dmZL9xosbz/4LHVB4TInf4+Zd5Hn/zfe+9/UGNIuAcAhBBbK4cO0TrnrKOCc8oYYZIzxhhPkpgBgNPWARJBLPb7zaTR5JwxITxTyWeX6zp/9IOXbwfpYFXhn/7ySeWMNtrzfEQsioIQwiinlBBHPMYpcAKcemEQBJxzrc0mzy262mAz6/QGndjn5WbD0VjiAJmvCLva1p89uXi7cBu3OV9svDg2BavqOgwDLnhV1xQYUCY4d8AdAeH5W22VyYMgcM7VBvNKxWkn3ekro548fizQAkFHHDImEHwr4mdXm3fe/f2j08uTi0leayRO+JJJGSZxI20SSrU2da2cI4wxrQ1jlBJXFtsi31Li0qzV6w+ms/nx8fHp0RNiLW+laVVt8lJJFhiDILz//ujLk4uLVa7n29IoEkWxQfQ8j0vpB5YB40JaAgYdReectVorrQLf77TbWWegHNSSl55ELvKq5HVVekBqqwWThhEHAEF8ejEBzox2xmBVVXmeA4DneZEUQeADoPS9IIyVMtP5HInhArJG1Gul/X5rmdeb5WK7Wqat1nQy5XVZeYyGnKAuKSNIEB0iYUY5Z6lzzjmHiACwWCzmumzEUTNrNRj4xLdYc2qZx+qq9jjl1JpiZYp6u5yhVr4nKsb+CkyFkScvikzRAAAAAElFTkSuQmCC"
+          "image":{
+            "b64": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAAw0lEQVR4nGNgGFggVVj4/y8Q2GOR83n+58/fP0DwcSqmpNN7oOTJw6f+/H2pjUU2JCSEk0EWqN0cl828e/FIxvz9/9cCh1zS5z9/G9mwyzl/+PNnKQ45nyNAr9ThMHQ/UG4tDofuB4bQIhz6fIBenMWJQ+7Vn7+zeLCbKXv6z59NOPQVgsIcW4QA9YFi6wNQLrKwsBebW/68DJ388Nun5XFocrqvIFH59+XhBAxThTfeB0r+vP/QHbuDCgr2JmOXoSsAAKK7bU3vISS4AAAAAElFTkSuQmCC"
           }
        }
     ]
diff --git a/test/e2e/predictor/test_triton.py b/test/e2e/predictor/test_triton.py
--- a/test/e2e/predictor/test_triton.py
+++ b/test/e2e/predictor/test_triton.py
@@ -33,13 +33,18 @@ def test_triton():
         min_replicas=1,
         triton=V1beta1TritonSpec(
             storage_uri='gs://kfserving-examples/models/torchscript',
-            ports=[V1ContainerPort(name="h2c", protocol="TCP", container_port=9000)]
+            ports=[V1ContainerPort(name="h2c", protocol="TCP", container_port=9000)],
+            resources=V1ResourceRequirements(
+                requests={'cpu': '100m', 'memory': '1Gi'},
+                limits={'cpu': '1', 'memory': '1Gi'},
+            ),
         )
     )
     transformer = V1beta1TransformerSpec(
         min_replicas=1,
         containers=[V1Container(
-                      image='kserve/grpc-image-transformer:latest',
+                      image='809251082950.dkr.ecr.us-west-2.amazonaws.com/kserve/image-transformer:'
+                            + os.environ.get("PULL_BASE_SHA"),
                       name='kserve-container',
                       resources=V1ResourceRequirements(
                           requests={'cpu': '100m', 'memory': '1Gi'},
diff --git a/test/e2e/transformer/test_raw_transformer.py b/test/e2e/transformer/test_raw_transformer.py
--- a/test/e2e/transformer/test_raw_transformer.py
+++ b/test/e2e/transformer/test_raw_transformer.py
@@ -12,9 +12,6 @@
 # limitations under the License.
 
 import os
-import json
-import requests
-import time
 import logging
 from kubernetes import client
 
@@ -28,7 +25,7 @@
 from kubernetes.client import V1ResourceRequirements
 from kubernetes.client import V1Container
 from kubernetes.client import V1EnvVar
-from ..common.utils import get_cluster_ip
+from ..common.utils import predict
 from ..common.utils import KSERVE_TEST_NAMESPACE
 logging.basicConfig(level=logging.INFO)
 kserve_client = KServeClient(
@@ -36,25 +33,27 @@
 
 
 def test_transformer():
-    service_name = 'raw'
+    service_name = 'raw-transformer'
     predictor = V1beta1PredictorSpec(
         min_replicas=1,
         pytorch=V1beta1TorchServeSpec(
             storage_uri='gs://kfserving-examples/models/torchserve/image_classifier',
             resources=V1ResourceRequirements(
-                requests={'cpu': '100m', 'memory': '2Gi'},
-                limits={'cpu': '100m', 'memory': '2Gi'}
+                requests={'cpu': '100m', 'memory': '1Gi'},
+                limits={'cpu': '1', 'memory': '1Gi'}
             )
         ),
     )
     transformer = V1beta1TransformerSpec(
         min_replicas=1,
         containers=[V1Container(
-            image='kserve/torchserve-image-transformer:latest',
+            image='809251082950.dkr.ecr.us-west-2.amazonaws.com/kserve/image-transformer:'
+                  + os.environ.get("PULL_BASE_SHA"),
             name='kserve-container',
             resources=V1ResourceRequirements(
-                requests={'cpu': '100m', 'memory': '2Gi'},
-                limits={'cpu': '100m', 'memory': '2Gi'}),
+                requests={'cpu': '100m', 'memory': '1Gi'},
+                limits={'cpu': '100m', 'memory': '1Gi'}),
+            args=["--model_name", "mnist"],
             env=[V1EnvVar(name="STORAGE_URI", value="gs://kfserving-examples/models/torchserve/image_classifier")])]
     )
 
@@ -72,31 +71,11 @@ def test_transformer():
         kserve_client.wait_isvc_ready(
             service_name, namespace=KSERVE_TEST_NAMESPACE)
     except RuntimeError as e:
+        print(kserve_client.api_instance.get_namespaced_custom_object("serving.knative.dev", "v1",
+                                                                      KSERVE_TEST_NAMESPACE,
+                                                                      "services", service_name + "-predictor-default"))
         raise e
 
-    time.sleep(30)
-
-    isvc = kserve_client.get(
-        service_name,
-        namespace=KSERVE_TEST_NAMESPACE,
-    )
-
-    cluster_ip = get_cluster_ip()
-    logging.info("clusterip = %s", cluster_ip)
-
-    host = isvc["status"]["url"]
-    host = host[host.rfind('/')+1:]
-    url = 'http://{}/v1/models/mnist:predict'.format(cluster_ip)
-    logging.info("url = %s ", url)
-    headers = {"Host": host}
-    data_str = '{"instances": [{"data": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAAw0lE\
-    QVR4nGNgGFggVVj4/y8Q2GOR83n+58/fP0DwcSqmpNN7oOTJw6f+/H2pjUU2JCSEk0EWqN0cl828e/FIxvz9/9cCh1\
-        zS5z9/G9mwyzl/+PNnKQ45nyNAr9ThMHQ/UG4tDofuB4bQIhz6fIBenMWJQ+7Vn7+zeLCbKXv6z59NOPQVgsIcW\
-            4QA9YFi6wNQLrKwsBebW/68DJ388Nun5XFocrqvIFH59+XhBAxThTfeB0r+vP/QHbuDCgr2JmOXoSsAAKK7b\
-                U3vISS4AAAAAElFTkSuQmCC", "target": 0}]}'
-    res = requests.post(url, data_str, headers=headers)
-    logging.info("res.text = %s", res.text)
-    preds = json.loads(res.content.decode("utf-8"))
-    assert(preds["predictions"] == [2])
-
+    res = predict(service_name, "./data/transformer.json", model_name="mnist")
+    assert(res.get("predictions")[0] == 2)
     kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
diff --git a/test/e2e/transformer/test_transformer.py b/test/e2e/transformer/test_transformer.py
--- a/test/e2e/transformer/test_transformer.py
+++ b/test/e2e/transformer/test_transformer.py
@@ -46,11 +46,13 @@ def test_transformer():
     transformer = V1beta1TransformerSpec(
         min_replicas=1,
         containers=[V1Container(
-                      image='809251082950.dkr.ecr.us-west-2.amazonaws.com/kserve/image-transformer:latest',
+                      image='809251082950.dkr.ecr.us-west-2.amazonaws.com/kserve/image-transformer:'
+                            + os.environ.get("PULL_BASE_SHA"),
                       name='kserve-container',
                       resources=V1ResourceRequirements(
                           requests={'cpu': '100m', 'memory': '1Gi'},
                           limits={'cpu': '100m', 'memory': '1Gi'}),
+                      args=["--model_name", "mnist"],
                       env=[V1EnvVar(name="STORAGE_URI",
                                     value="gs://kfserving-examples/models/torchserve/image_classifier")])]
     )
@@ -74,6 +76,6 @@ def test_transformer():
         for pod in pods.items:
             print(pod)
         raise e
-    res = predict(service_name, "./data/torchserve_input.json", model_name="mnist")
+    res = predict(service_name, "./data/transformer.json", model_name="mnist")
     assert(res.get("predictions")[0] == 2)
     kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
diff --git a/test/scripts/post-e2e-tests.sh b/test/scripts/post-e2e-tests.sh
--- a/test/scripts/post-e2e-tests.sh
+++ b/test/scripts/post-e2e-tests.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2019 The Kubeflow Authors.
+# Copyright 2021 The KServe Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,5 +24,9 @@ echo "Configuring kubectl ..."
 pip3 install awscli --upgrade --user
 aws eks update-kubeconfig --region=${AWS_REGION} --name=${CLUSTER_NAME}
 
+# Print e2e test events
+kubectl get events -n kserve-ci-e2e-test
+
 # Print controller logs
 kubectl logs kserve-controller-manager-0 -n kserve manager
+
diff --git a/test/scripts/run-e2e-tests.sh b/test/scripts/run-e2e-tests.sh
--- a/test/scripts/run-e2e-tests.sh
+++ b/test/scripts/run-e2e-tests.sh
@@ -145,6 +145,3 @@ echo "Starting E2E functional tests ..."
 pushd test/e2e >/dev/null
   pytest -n 4 --ignore=credentials/test_set_creds.py
 popd
-
-kubectl get events -n kserve-ci-e2e-test
-kubectl get events -n kserve
diff --git a/test/workflows/components/workflows.libsonnet b/test/workflows/components/workflows.libsonnet
--- a/test/workflows/components/workflows.libsonnet
+++ b/test/workflows/components/workflows.libsonnet
@@ -381,9 +381,9 @@
             ]),  // build-agent
             $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("build-custom-image-transformer", kanikoExecutorImage, [
               "/kaniko/executor",
-              "--dockerfile=" + srcDir + "/docs/samples/v1beta1/transformer/torchserve_image_transformer/transformer.Dockerfile",
-              "--context=dir://" + srcDir + "/docs/samples/v1beta1/transformer/torchserve_image_transformer",
-              "--destination=" + "809251082950.dkr.ecr.us-west-2.amazonaws.com/kserve/image-transformer:latest",
+              "--dockerfile=" + srcDir + "/python/custom_transformer.Dockerfile",
+              "--context=dir://" + srcDir + "/python",
+              "--destination=" + "809251082950.dkr.ecr.us-west-2.amazonaws.com/kserve/image-transformer:$(PULL_BASE_SHA)",
             ]),  // build-custom-image-transformer
             $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("build-pytorchserver", kanikoExecutorImage, [
               "/kaniko/executor",