Repository: kserve/kserve

Issue:
Harden Puller logic in the side-car Agent
/kind feature

**Describe the solution you'd like**
With the merging of #989 a variety of enhancements are still required before we can remove the [POC] label from the multi-model puller. These enhancements include but are not limited to:
* Handle retries
* Move logger to zapr
* Set proper buffer size
* Handle request logic
* Implement a variety of protocols: HDFS, GCS, FILE, HTTPS
* Handle proper config overrides of s3 clients
* Remove need for timestamp for config diffing by using a bool
* Figure out if mid-download failures can be checkpointed and recovered *optional
* loop through map in a parallel way instead of sequential *optional
* Handle load, unload, load events *optional



**Anything else you would like to add:**
* Unit tests
* End to End tests


Discussion:
Issue-Label Bot is automatically applying the labels:

| Label  | Probability |
| ------------- | ------------- |
| area/engprod | 0.57 |

Please mark this comment with :thumbsup: or :thumbsdown: to give our bot feedback! 
Links: [app homepage](https://github.com/marketplace/issue-label-bot), [dashboard](https://label-bot-prod.mlbot.net/data/kubeflow/kfserving) and [code](https://github.com/hamelsmu/MLapp) for this bot.
Issue Label Bot is not confident enough to auto-label this issue.
                See [dashboard](https://label-bot-prod.mlbot.net/data/kubeflow/kfserving) for more details.
                

Resolving patch:
diff --git a/python/kfserving/kfserving/handlers/http.py b/python/kfserving/kfserving/handlers/http.py
--- a/python/kfserving/kfserving/handlers/http.py
+++ b/python/kfserving/kfserving/handlers/http.py
@@ -35,10 +35,11 @@ def get_model(self, name: str):
         return model
 
     def validate(self, request):
-        if "instances" in request and not isinstance(request["instances"], list):
+        if ("instances" in request and not isinstance(request["instances"], list)) or \
+           ("inputs" in request and not isinstance(request["inputs"], list)):
             raise tornado.web.HTTPError(
                 status_code=HTTPStatus.BAD_REQUEST,
-                reason="Expected \"instances\" to be a list"
+                reason="Expected \"instances\" or \"inputs\" to be a list"
             )
         return request
 
diff --git a/python/kfserving/kfserving/kfserver.py b/python/kfserving/kfserving/kfserver.py
--- a/python/kfserving/kfserving/kfserver.py
+++ b/python/kfserving/kfserving/kfserver.py
@@ -62,18 +62,27 @@ def create_application(self):
         return tornado.web.Application([
             # Server Liveness API returns 200 if server is alive.
             (r"/", LivenessHandler),
+            (r"/v2/health/live", LivenessHandler),
             (r"/v1/models",
              ListHandler, dict(models=self.registered_models)),
+            (r"/v2/models",
+             ListHandler, dict(models=self.registered_models)),
             # Model Health API returns 200 if model is ready to serve.
             (r"/v1/models/([a-zA-Z0-9_-]+)",
              HealthHandler, dict(models=self.registered_models)),
+            (r"/v2/models/([a-zA-Z0-9_-]+)/status",
+             HealthHandler, dict(models=self.registered_models)),
             (r"/v1/models/([a-zA-Z0-9_-]+):predict",
              PredictHandler, dict(models=self.registered_models)),
+            (r"/v2/models/([a-zA-Z0-9_-]+)/infer",
+             PredictHandler, dict(models=self.registered_models)),
             (r"/v1/models/([a-zA-Z0-9_-]+):explain",
              ExplainHandler, dict(models=self.registered_models)),
-            (r"/v1/models/([a-zA-Z0-9_-]+)/load",
+            (r"/v2/models/([a-zA-Z0-9_-]+)/explain",
+             ExplainHandler, dict(models=self.registered_models)),
+            (r"/v2/repository/models/([a-zA-Z0-9_-]+)/load",
              LoadHandler, dict(models=self.registered_models)),
-            (r"/v1/models/([a-zA-Z0-9_-]+)/unload",
+            (r"/v2/repository/models/([a-zA-Z0-9_-]+)/unload",
              UnloadHandler, dict(models=self.registered_models)),
         ])
 


Test patch:
diff --git a/pkg/agent/agent_suite_test.go b/pkg/agent/agent_suite_test.go
new file mode 100644
--- /dev/null
+++ b/pkg/agent/agent_suite_test.go
@@ -0,0 +1,13 @@
+package agent_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestAgent(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Agent Suite")
+}
diff --git a/pkg/agent/watcher_test.go b/pkg/agent/watcher_test.go
new file mode 100644
--- /dev/null
+++ b/pkg/agent/watcher_test.go
@@ -0,0 +1,273 @@
+package agent
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/golang/protobuf/proto"
+	"github.com/kubeflow/kfserving/pkg/agent/storage"
+	"github.com/kubeflow/kfserving/pkg/apis/serving/v1beta1"
+	"github.com/kubeflow/kfserving/pkg/modelconfig"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"io/ioutil"
+	logger "log"
+	"os"
+)
+
+type mockS3Client struct {
+	s3iface.S3API
+}
+
+func (m *mockS3Client) ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	return &s3.ListObjectsOutput{
+		Contents: []*s3.Object{
+			{
+				Key: proto.String("model.pt"),
+			},
+		},
+	}, nil
+}
+
+type mockS3Downloder struct {
+}
+
+func (m *mockS3Downloder) DownloadWithIterator(aws.Context, s3manager.BatchDownloadIterator, ...func(*s3manager.Downloader)) error {
+	return nil
+}
+
+type mockS3FailDownloder struct {
+	err error
+}
+
+func (m *mockS3FailDownloder) DownloadWithIterator(aws.Context, s3manager.BatchDownloadIterator, ...func(*s3manager.Downloader)) error {
+	var errs []s3manager.Error
+	errs = append(errs, s3manager.Error{
+		OrigErr: fmt.Errorf("failed to download"),
+		Bucket:  aws.String("modelRepo"),
+		Key:     aws.String("model1/model.pt"),
+	})
+	return s3manager.NewBatchError("BatchedDownloadIncomplete", "some objects have failed to download.", errs)
+}
+
+var _ = Describe("Watcher", func() {
+	var modelDir string
+	BeforeEach(func() {
+		dir, err := ioutil.TempDir("", "example")
+		if err != nil {
+			logger.Fatal(err)
+		}
+		modelDir = dir
+		logger.Printf("Creating temp dir %v\n", modelDir)
+	})
+	AfterEach(func() {
+		os.RemoveAll(modelDir)
+		logger.Printf("Deleted temp dir %v\n", modelDir)
+	})
+	Describe("Sync model config", func() {
+		Context("Sync new models", func() {
+			It("should download the new models", func() {
+				defer GinkgoRecover()
+				logger.Printf("Sync model config using temp dir %v\n", modelDir)
+				watcher := NewWatcher("/tmp/configs", modelDir)
+				puller := Puller{
+					channelMap:  make(map[string]*ModelChannel),
+					completions: make(chan *ModelOp, 4),
+					opStats:     make(map[string]map[OpType]int),
+					Downloader: Downloader{
+						ModelDir: modelDir + "/test1",
+						Providers: map[storage.Protocol]storage.Provider{
+							storage.S3: &storage.S3Provider{
+								Client:     &mockS3Client{},
+								Downloader: &mockS3Downloder{},
+							},
+						},
+					},
+				}
+				go puller.processCommands(watcher.ModelEvents)
+				modelConfigs := modelconfig.ModelConfigs{
+					{
+						Name: "model1",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model1",
+							Framework:  "sklearn",
+						},
+					},
+					{
+						Name: "model2",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model2",
+							Framework:  "sklearn",
+						},
+					},
+				}
+				watcher.parseConfig(modelConfigs)
+				Eventually(func() int { return len(puller.channelMap) }).Should(Equal(0))
+				Eventually(func() int { return puller.opStats["model1"][Add] }).Should(Equal(1))
+				Eventually(func() int { return puller.opStats["model2"][Add] }).Should(Equal(1))
+			})
+		})
+
+		Context("Sync delete models", func() {
+			It("should download the deleted models", func() {
+				defer GinkgoRecover()
+				logger.Printf("Sync delete models using temp dir %v\n", modelDir)
+				watcher := NewWatcher("/tmp/configs", modelDir)
+				puller := Puller{
+					channelMap:  make(map[string]*ModelChannel),
+					completions: make(chan *ModelOp, 4),
+					opStats:     make(map[string]map[OpType]int),
+					Downloader: Downloader{
+						ModelDir: modelDir + "/test2",
+						Providers: map[storage.Protocol]storage.Provider{
+							storage.S3: &storage.S3Provider{
+								Client:     &mockS3Client{},
+								Downloader: &mockS3Downloder{},
+							},
+						},
+					},
+				}
+				go puller.processCommands(watcher.ModelEvents)
+				modelConfigs := modelconfig.ModelConfigs{
+					{
+						Name: "model1",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model1",
+							Framework:  "sklearn",
+						},
+					},
+					{
+						Name: "model2",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model2",
+							Framework:  "sklearn",
+						},
+					},
+				}
+				watcher.parseConfig(modelConfigs)
+				// remove model2
+				modelConfigs = modelconfig.ModelConfigs{
+					{
+						Name: "model1",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model1",
+							Framework:  "sklearn",
+						},
+					},
+				}
+				watcher.parseConfig(modelConfigs)
+				Eventually(func() int { return len(puller.channelMap) }).Should(Equal(0))
+				Eventually(func() int { return puller.opStats["model1"][Add] }).Should(Equal(1))
+				Eventually(func() int { return puller.opStats["model2"][Add] }).Should(Equal(1))
+				Eventually(func() int { return puller.opStats["model2"][Remove] }).Should(Equal(1))
+			})
+		})
+
+		Context("Sync update models", func() {
+			It("should update models", func() {
+				defer GinkgoRecover()
+				logger.Printf("Sync update models using temp dir %v\n", modelDir)
+				watcher := NewWatcher("/tmp/configs", modelDir)
+				puller := Puller{
+					channelMap:  make(map[string]*ModelChannel),
+					completions: make(chan *ModelOp, 4),
+					opStats:     make(map[string]map[OpType]int),
+					Downloader: Downloader{
+						ModelDir: modelDir + "/test3",
+						Providers: map[storage.Protocol]storage.Provider{
+							storage.S3: &storage.S3Provider{
+								Client:     &mockS3Client{},
+								Downloader: &mockS3Downloder{},
+							},
+						},
+					},
+				}
+				go puller.processCommands(watcher.ModelEvents)
+				modelConfigs := modelconfig.ModelConfigs{
+					{
+						Name: "model1",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model1",
+							Framework:  "sklearn",
+						},
+					},
+					{
+						Name: "model2",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model2",
+							Framework:  "sklearn",
+						},
+					},
+				}
+				watcher.parseConfig(modelConfigs)
+				// remove model2
+				modelConfigs = modelconfig.ModelConfigs{
+					{
+						Name: "model1",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model1",
+							Framework:  "sklearn",
+						},
+					},
+					{
+						Name: "model2",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model2v2",
+							Framework:  "sklearn",
+						},
+					},
+				}
+				watcher.parseConfig(modelConfigs)
+				Eventually(func() int { return len(puller.channelMap) }).Should(Equal(0))
+				Eventually(func() int { return puller.opStats["model1"][Add] }).Should(Equal(1))
+				Eventually(func() int { return puller.opStats["model2"][Add] }).Should(Equal(2))
+				Eventually(func() int { return puller.opStats["model2"][Remove] }).Should(Equal(1))
+			})
+		})
+
+		Context("Model download failure", func() {
+			It("should not create success file", func() {
+				defer GinkgoRecover()
+				logger.Printf("Using temp dir %v\n", modelDir)
+				var errs []s3manager.Error
+				errs = append(errs, s3manager.Error{
+					OrigErr: fmt.Errorf("failed to download"),
+					Bucket:  aws.String("modelRepo"),
+					Key:     aws.String("model1/model.pt"),
+				})
+				var err error
+				err = s3manager.NewBatchError("BatchedDownloadIncomplete", "some objects have failed to download.", errs)
+				watcher := NewWatcher("/tmp/configs", modelDir)
+				puller := Puller{
+					channelMap:  make(map[string]*ModelChannel),
+					completions: make(chan *ModelOp, 4),
+					opStats:     make(map[string]map[OpType]int),
+					Downloader: Downloader{
+						ModelDir: modelDir + "/test4",
+						Providers: map[storage.Protocol]storage.Provider{
+							storage.S3: &storage.S3Provider{
+								Client:     &mockS3Client{},
+								Downloader: &mockS3FailDownloder{err: err},
+							},
+						},
+					},
+				}
+				go puller.processCommands(watcher.ModelEvents)
+				modelConfigs := modelconfig.ModelConfigs{
+					{
+						Name: "model1",
+						Spec: v1beta1.ModelSpec{
+							StorageURI: "s3://models/model1",
+							Framework:  "sklearn",
+						},
+					},
+				}
+				watcher.parseConfig(modelConfigs)
+				Eventually(func() int { return len(puller.channelMap) }).Should(Equal(0))
+				Eventually(func() int { return puller.opStats["model1"][Add] }).Should(Equal(1))
+			})
+		})
+	})
+})
diff --git a/python/kfserving/test/test_server.py b/python/kfserving/test/test_server.py
--- a/python/kfserving/test/test_server.py
+++ b/python/kfserving/test/test_server.py
@@ -97,13 +97,13 @@ def app(self):  # pylint: disable=no-self-use
         return server.create_application()
 
     async def test_load(self, http_server_client):
-        resp = await http_server_client.fetch('/v1/models/model/load',
+        resp = await http_server_client.fetch('/v2/repository/models/model/load',
                                               method="POST", body=b'')
         assert resp.code == 200
         assert resp.body == b'{"name": "model", "load": true}'
 
     async def test_unload(self, http_server_client):
-        resp = await http_server_client.fetch('/v1/models/model/unload',
+        resp = await http_server_client.fetch('/v2/repository/models/model/unload',
                                               method="POST", body=b'')
         assert resp.code == 200
         assert resp.body == b'{"name": "model", "unload": true}'
@@ -117,13 +117,13 @@ def app(self):  # pylint: disable=no-self-use
 
     async def test_load_fail(self, http_server_client):
         with pytest.raises(HTTPClientError) as err:
-            _ = await http_server_client.fetch('/v1/models/model/load',
+            _ = await http_server_client.fetch('/v2/repository/models/model/load',
                                                method="POST", body=b'')
         assert err.value.code == 503
 
     async def test_unload_fail(self, http_server_client):
         with pytest.raises(HTTPClientError) as err:
-            _ = await http_server_client.fetch('/v1/models/model/unload',
+            _ = await http_server_client.fetch('/v2/repository/models/model/unload',
                                                method="POST", body=b'')
         assert err.value.code == 404