Repository: Xilinx/inference-server (language: python)

Issue:
Properly support multiple input tensors
There is partial support in the batcher(s) and none in the predict_api request construction


Discussion / hints:

Fix patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index dcb6a78e1..85a70d2a4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -19,13 +19,23 @@ repos:
     rev: v0.6.10
     hooks:
     - id: cmake-format
+  - repo: https://github.com/cmake-lint/cmake-lint
+    rev: 1.4.2
+    hooks:
+      - id: cmakelint
+        # cmake-format may align text vertically at weird indents
+        args: [--filter=-whitespace/indent]
+  # - repo: https://github.com/Takishima/cmake-pre-commit-hooks
+  #   rev: v1.5.3
+  #   hooks:
+  #     - id: lizard
   # general hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.1.0
     hooks:
       - id: check-added-large-files
       - id: pretty-format-json
-        args: [--autofix]
+        args: [--autofix, --no-sort-keys]
       - id: check-merge-conflict
       - id: trailing-whitespace
       - id: end-of-file-fixer
diff --git a/Dockerfile b/Dockerfile
index 2d31df5a9..26479c962 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -142,6 +142,7 @@ WORKDIR /tmp
 RUN rm -rf ${COPY_DIR} && mkdir ${COPY_DIR} && mkdir -p ${MANIFESTS_DIR}
 
 # install protobuf 3.19.4 for gRPC - Used by Vitis AI and onnx
+# onnx requires 3.19.4 due to Protobuf python
 RUN apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \
         autoconf \
@@ -151,10 +152,10 @@ RUN apt-get update \
         unzip \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* \
-    && VERSION=3.21.1 \
-    && wget --quiet https://github.com/protocolbuffers/protobuf/releases/download/v21.1/protobuf-cpp-${VERSION}.tar.gz \
+    && VERSION=3.19.4 \
+    && wget --quiet https://github.com/protocolbuffers/protobuf/releases/download/v${VERSION}/protobuf-cpp-${VERSION}.tar.gz \
     && tar -xzf protobuf-cpp-${VERSION}.tar.gz \
-    && cd protobuf-${VERSION} \
+    && cd protobuf-${VERSION}/cmake \
     && cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=NO -DBUILD_SHARED_LIBS=YES \
     && cmake --build build --target install -- -j$(($(nproc) - 1)) \
     && cat build/install_manifest.txt | xargs -i bash -c "if [ -f {} ]; then cp --parents -P {} ${COPY_DIR}; fi" \
diff --git a/docs/dependencies.rst b/docs/dependencies.rst
index 076107ac4..8eee1bc2c 100644
--- a/docs/dependencies.rst
+++ b/docs/dependencies.rst
@@ -170,7 +170,7 @@ The following packages are installed from Github.
     :github:`open-telemetry/opentelemetry-cpp`,1.1.0,Apache 2.0,Dynamically linked by proteus-server\ :superscript:`a 0`
     :github:`pybind/pybind11`,2.9.1,BSD-3,Headers used to build Python bindings\ :superscript:`d 0`
     :github:`jupp0r/prometheus-cpp`,0.12.2,MIT,Dynamically linked by proteus-server for metrics\ :superscript:`a 0`
-    :github:`protocolbuffers/protobuf`,3.21.1,BSD-3,Dynamically linked by proteus-server and Vitis libraries\ :superscript:`a 0`
+    :github:`protocolbuffers/protobuf`,3.19.4,BSD-3,Dynamically linked by proteus-server and Vitis libraries\ :superscript:`a 0`
     :github:`fpagliughi/sockpp`,e5c51b5,BSD-3,Dynamically linked by proteus-server :superscript:`a 0`
     :github:`gabime/spdlog`,1.8.2,MIT,Statically linked by proteus-server for logging\ :superscript:`a 0`
     :github:`Xilinx/Vitis-AI`,latest,Apache 2.0,VART is dynamically linked by proteus-server\ :superscript:`a 1`
diff --git a/examples/cpp/custom_processing.cpp b/examples/cpp/custom_processing.cpp
index 9dcbf94ea..b2d8540a0 100644
--- a/examples/cpp/custom_processing.cpp
+++ b/examples/cpp/custom_processing.cpp
@@ -12,23 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <algorithm>              // for max
+#include <algorithm>              // for max, fill
 #include <array>                  // for array
 #include <cmath>                  // for exp
-#include <cstdint>                // for int8_t, uint64_t
-#include <cstdlib>                // for getenv, size_t
-#include <future>                 // IWYU pragma: keep
+#include <cstdint>                // for int8_t, uint64_t, int32_t
+#include <cstdlib>                // for exit, getenv, size_t
 #include <initializer_list>       // for initializer_list
 #include <iostream>               // for operator<<, basic_ostream::operator<<
-#include <memory>                 // for allocator_traits<>::value_type
+#include <memory>                 // for allocator, allocator_traits<>::valu...
 #include <opencv2/core.hpp>       // for Mat, Vec3b, MatSize, Vec, CV_8SC3
 #include <opencv2/imgcodecs.hpp>  // for imread
 #include <opencv2/imgproc.hpp>    // for resize
 #include <queue>                  // for priority_queue
-#include <string>                 // for string, allocator, operator==, basi...
+#include <string>                 // for string, operator==, basic_string
+#include <unordered_set>          // for operator==, unordered_set, unordere...
 #include <utility>                // for pair
 #include <vector>                 // for vector
 
+#ifdef ENABLE_GRPC
+#include <chrono>  // for seconds
+#include <thread>  // for sleep_for
+#endif
+
 // +include:
 #include "proteus/proteus.hpp"
 // -include:
@@ -46,8 +51,8 @@ std::vector<std::vector<int8_t>> preprocess(
   const auto width = 224;
   const auto channels = 3;
   std::string layout = "NHWC";
-  const auto fix_scale = 1;
-  const std::array mean = {123, 107, 104};
+  const int8_t fix_scale = 1;
+  const std::array<int8_t, 3> mean = {123, 107, 104};
 
   std::vector<std::vector<int8_t>> outputs;
   outputs.reserve(paths.size());
@@ -68,7 +73,9 @@ std::vector<std::vector<int8_t>> preprocess(
         for (int h = 0; h < height; h++) {
           for (int w = 0; w < width; w++) {
             output[(c * height * width) + (h * width) + w] =
-              (resizedImg.at<cv::Vec3b>(h, w)[c] - mean.at(c)) * fix_scale;
+              (static_cast<int8_t>(resizedImg.at<cv::Vec3b>(h, w)[c]) -
+               mean.at(c)) *
+              fix_scale;
           }
         }
       }
@@ -76,8 +83,10 @@ std::vector<std::vector<int8_t>> preprocess(
       for (int h = 0; h < height; h++) {
         for (int w = 0; w < width; w++) {
           for (int c = 0; c < channels; c++) {
-            output[h * width * channels + w * channels + c] =
-              (resizedImg.at<cv::Vec3b>(h, w)[c] - mean.at(c)) * fix_scale;
+            output[(h * width * channels) + (w * channels) + c] =
+              (static_cast<int8_t>(resizedImg.at<cv::Vec3b>(h, w)[c]) -
+               mean.at(c)) *
+              fix_scale;
           }
         }
       }
@@ -153,7 +162,7 @@ std::vector<int> postprocess(proteus::InferenceResponseOutput& output, int k) {
   auto size = output.getSize();
 
   std::vector<double> softmax;
-  softmax.resize(size);
+  softmax.resize(size, 0);
 
   calc_softmax<T>(data->data(), size, softmax.data());
   return get_top_k(softmax.data(), size, k);
@@ -256,7 +265,9 @@ int main() {
     "resnet_v1_50_tf.xmodel";
   const auto path_to_image = root + "/tests/assets/dog-3619020_640.jpg";
   // for this image, we know what we expect to receive with this XModel
-  const std::array gold_response_output = {259, 261, 260, 230, 154};
+  // note this is different than custom_processing.py due to how the top_k are
+  // calculated. Index 230 and 154 are equal classifications
+  const std::array gold_response_output = {259, 261, 260, 157, 230};
   const auto k = gold_response_output.size();
   // -user variables:
 
@@ -272,10 +283,7 @@ int main() {
   // +prepare images:
   std::vector<std::string> paths;
   paths.reserve(request_num);
-
-  for (auto i = 0; i < request_num; i++) {
-    paths.emplace_back(path_to_image);
-  }
+  paths.emplace_back(path_to_image);
 
   auto images = preprocess(paths);
   // -prepare images:
@@ -284,27 +292,27 @@ int main() {
   const std::initializer_list<uint64_t> shape = {224, 224, 3};
 
   proteus::InferenceRequest request;
-  for (auto i = 0; i < request_num; i++) {
-    request.addInputTensor(static_cast<void*>(images[i].data()), shape,
-                           proteus::DataType::INT8);
-  }
+  request.addInputTensor(static_cast<void*>(images[0].data()), shape,
+                         proteus::DataType::INT8);
   // -construct request:
 
-  auto results = infer(worker_name, request);
-
-  // +validate:
-  auto outputs = results.getOutputs();
-  for (auto& output : outputs) {
-    std::vector<int> top_k = postprocess(output, k);
-    for (size_t j = 0; j < k; j++) {
-      if (top_k[j] != gold_response_output.at(j)) {
-        std::cerr << "Output (" << top_k[j] << ") does not match golden ("
-                  << gold_response_output.at(j) << ")\n";
-        return 1;
+  for (auto i = 0; i < request_num; i++) {
+    auto results = infer(worker_name, request);
+
+    // +validate:
+    auto outputs = results.getOutputs();
+    for (auto& output : outputs) {
+      std::vector<int> top_k = postprocess(output, k);
+      for (size_t j = 0; j < k; j++) {
+        if (top_k[j] != gold_response_output.at(j)) {
+          std::cerr << "Output (" << top_k[j] << ") does not match golden ("
+                    << gold_response_output.at(j) << ")\n";
+          return 1;
+        }
       }
     }
+    // -validate:
   }
-  // -validate:
 
   std::cout << "custom_processing.cpp: Passed\n";
 
diff --git a/examples/cpp/pt_zendnn_client.cpp b/examples/cpp/pt_zendnn_client.cpp
index 16c5a520c..740b4eb26 100644
--- a/examples/cpp/pt_zendnn_client.cpp
+++ b/examples/cpp/pt_zendnn_client.cpp
@@ -12,20 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <algorithm>              // for max, copy
 #include <array>                  // for array
+#include <chrono>                 // for milliseconds, duration_cast, operator-
 #include <cmath>                  // for exp
-#include <cstdint>                // for int8_t, uint64_t
-#include <cstdlib>                // for getenv, size_t
-#include <fstream>                // For files
-#include <future>                 // for future
+#include <cstdint>                // for uint64_t
+#include <cstdlib>                // for exit, getenv, size_t
+#include <fstream>                // IWYU pragma: keep
 #include <initializer_list>       // for initializer_list
-#include <iostream>               // for operator<<, basic_ostream::operator<<
-#include <opencv2/core.hpp>       // for Mat, Vec3b, MatSize, Vec, CV_8SC3
+#include <iostream>               // for operator<<, basic_ostream, endl
+#include <memory>                 // for allocator_traits<>::value_type
+#include <opencv2/core.hpp>       // for Mat, randu, CV_32FC3, MatSize, Scalar
 #include <opencv2/imgcodecs.hpp>  // for imread
-#include <opencv2/imgproc.hpp>    // for resize
+#include <opencv2/imgproc.hpp>    // for cvtColor, resize, COLOR_BGR2RGB
 #include <queue>                  // for priority_queue, queue
-#include <string>                 // for string, allocator, operator==, basi...
-#include <utility>                // for pair, move
+#include <string>                 // for string, operator+, allocator, opera...
+#include <unordered_set>          // for operator==, unordered_set, unordere...
+#include <utility>                // for pair
 #include <vector>                 // for vector
 
 #include "proteus/proteus.hpp"  // for InferenceResponseFuture, terminate
@@ -260,8 +263,8 @@ int main() {
     auto images = preprocess(paths, options.input_size, options.resize_method);
 
     const std::initializer_list<uint64_t> shape = {
-      3, static_cast<long unsigned>(options.input_size),
-      static_cast<long unsigned>(options.input_size)};
+      3, static_cast<uint64_t>(options.input_size),
+      static_cast<uint64_t>(options.input_size)};
     std::queue<proteus::InferenceResponseFuture> queue;
     proteus::InferenceRequest request;
     request.addInputTensor(static_cast<void*>(images[0].data()), shape,
@@ -302,28 +305,28 @@ int main() {
 
     // inference
     const std::initializer_list<uint64_t> shape = {
-      3, static_cast<long unsigned>(options.input_size),
-      static_cast<long unsigned>(options.input_size)};
+      3, static_cast<uint64_t>(options.input_size),
+      static_cast<uint64_t>(options.input_size)};
     // Warmup laps to get the best performance
     for (int step = 0; step < options.warmup_step; step++) {
-      proteus::InferenceRequest request;
       for (auto i = 0; i < options.batch_size; i++) {
+        proteus::InferenceRequest request;
         request.addInputTensor(static_cast<void*>(images[i].data()), shape,
                                proteus::DataType::FP32);
+        auto results = client.modelInfer(workerName, request);
       }
-      auto results = client.modelInfer(workerName, request);
     }
 
     // Running for `steps` number of time for proper benchmarking
     auto start = std::chrono::high_resolution_clock::now();  // Timing the start
     for (int step = 0; step < options.steps; step++) {
       std::queue<proteus::InferenceResponseFuture> queue;
-      proteus::InferenceRequest request;
       for (auto i = 0; i < options.batch_size; i++) {
+        proteus::InferenceRequest request;
         request.addInputTensor(static_cast<void*>(images[i].data()), shape,
                                proteus::DataType::FP32);
+        auto results = client.modelInfer(workerName, request);
       }
-      auto results = client.modelInfer(workerName, request);
     }
     // Timing the prediction
     auto stop = std::chrono::high_resolution_clock::now();
@@ -332,11 +335,11 @@ int main() {
     time_tmp = duration.count() / options.steps;
   }
 
-  if (!options.image_location.empty())
+  if (!options.image_location.empty()) {
     std::cout << "\nTime taken for " + options.image_location + " : " +
                    std::to_string(time_tmp)
               << "ms" << std::endl;
-  else {
+  } else {
     std::cout << "\nAverage time taken for " +
                    std::to_string(options.batch_size) +
                    " images: " + std::to_string(time_tmp)
diff --git a/examples/cpp/tf_zendnn_client.cpp b/examples/cpp/tf_zendnn_client.cpp
index 75f99ed1c..ae65950c5 100644
--- a/examples/cpp/tf_zendnn_client.cpp
+++ b/examples/cpp/tf_zendnn_client.cpp
@@ -17,20 +17,22 @@
  * @brief Implements a client application for TfZendnn worker
  */
 
-#include <array>                  // for array
+#include <algorithm>              // for copy, max
+#include <chrono>                 // for milliseconds, duration_cast, operator-
 #include <cmath>                  // for exp
-#include <cstdint>                // for int8_t, uint64_t
-#include <cstdlib>                // for getenv, size_t
-#include <fstream>                // For files
-#include <future>                 // for future
+#include <cstdint>                // for uint64_t
+#include <cstdlib>                // for exit, getenv, size_t
+#include <fstream>                // IWYU pragma: keep
 #include <initializer_list>       // for initializer_list
-#include <iostream>               // for operator<<, basic_ostream::operator<<
-#include <opencv2/core.hpp>       // for Mat, Vec3b, MatSize, Vec, CV_8SC3
+#include <iostream>               // for operator<<, basic_ostream, endl
+#include <memory>                 // for allocator_traits<>::value_type
+#include <opencv2/core.hpp>       // for Mat, MatSize, operator*, operator-
 #include <opencv2/imgcodecs.hpp>  // for imread
-#include <opencv2/imgproc.hpp>    // for resize
+#include <opencv2/imgproc.hpp>    // for resize, cvtColor, INTER_LINEAR, COL...
 #include <queue>                  // for priority_queue, queue
-#include <string>                 // for string, allocator, operator==, basi...
-#include <utility>                // for pair, move
+#include <string>                 // for string, operator+, allocator, opera...
+#include <unordered_set>          // for operator==, unordered_set, unordere...
+#include <utility>                // for pair
 #include <vector>                 // for vector
 
 #include "proteus/proteus.hpp"  // for InferenceResponseFuture, terminate
@@ -275,8 +277,8 @@ int main() {
     auto images = preprocess(paths, options.input_size, options.resize_method);
 
     const std::initializer_list<uint64_t> shape = {
-      static_cast<long unsigned>(options.input_size),
-      static_cast<long unsigned>(options.input_size), 3};
+      static_cast<uint64_t>(options.input_size),
+      static_cast<uint64_t>(options.input_size), 3};
     std::queue<proteus::InferenceResponseFuture> queue;
     proteus::InferenceRequest request;
     auto start = std::chrono::high_resolution_clock::now();  // Timing the start
@@ -317,29 +319,29 @@ int main() {
 
     // inference
     const std::initializer_list<uint64_t> shape = {
-      static_cast<long unsigned>(options.input_size),
-      static_cast<long unsigned>(options.input_size), 3};
+      static_cast<uint64_t>(options.input_size),
+      static_cast<uint64_t>(options.input_size), 3};
     // Warmup laps to get the best performance
     for (int step = 0; step < options.warmup_step; step++) {
       std::queue<proteus::InferenceResponseFuture> queue;
-      proteus::InferenceRequest request;
       for (auto i = 0; i < options.batch_size; i++) {
+        proteus::InferenceRequest request;
         request.addInputTensor(static_cast<void*>(images[i].data()), shape,
                                proteus::DataType::FP32);
+        auto results = client.modelInfer(workerName, request);
       }
-      auto results = client.modelInfer(workerName, request);
     }
 
     // Running for `steps` number of time for proper benchmarking
     auto start = std::chrono::high_resolution_clock::now();  // Timing the start
     for (int step = 0; step < options.steps; step++) {
       std::queue<proteus::InferenceResponseFuture> queue;
-      proteus::InferenceRequest request;
       for (auto i = 0; i < options.batch_size; i++) {
+        proteus::InferenceRequest request;
         request.addInputTensor(static_cast<void*>(images[i].data()), shape,
                                proteus::DataType::FP32);
+        auto results = client.modelInfer(workerName, request);
       }
-      auto results = client.modelInfer(workerName, request);
     }
     // Timing the prediction
     auto stop = std::chrono::high_resolution_clock::now();
@@ -348,11 +350,11 @@ int main() {
     time_tmp = duration.count() / options.steps;
   }
 
-  if (!options.image_location.empty())
+  if (!options.image_location.empty()) {
     std::cout << "\nTime taken for " + options.image_location + " : " +
                    std::to_string(time_tmp)
               << "ms" << std::endl;
-  else {
+  } else {
     std::cout << "\nAverage time taken for " +
                    std::to_string(options.batch_size) +
                    " images: " + std::to_string(time_tmp)
diff --git a/examples/python/custom_processing.py b/examples/python/custom_processing.py
index caa7502d5..860d8e071 100644
--- a/examples/python/custom_processing.py
+++ b/examples/python/custom_processing.py
@@ -123,7 +123,7 @@ def main():
 
     # +get images:
     images = []
-    for _ in range(batch_size):
+    for _ in range(1):
         image = cv2.imread(path_to_image)
         images.append(image)
     # -get images:
@@ -131,16 +131,17 @@ def main():
     # +inference:
     images = preprocess(images)
     # Construct the request and send it
-    request = proteus.ImageInferenceRequest(images, True)
-    response = client.modelInfer(worker_name, request)
-    assert not response.isError(), response.getError()
-    outputs = response.getOutputs()
-    for output in outputs:
-        assert output.datatype == proteus.DataType.INT8
-        recv_data = output.getInt8Data()
-        # Can optionally post-process the result
-        k = postprocess(recv_data, 5)
-        assert k == gold_response_output
+    for _ in range(batch_size):
+        request = proteus.ImageInferenceRequest(images, True)
+        response = client.modelInfer(worker_name, request)
+        assert not response.isError(), response.getError()
+        outputs = response.getOutputs()
+        for output in outputs:
+            assert output.datatype == proteus.DataType.INT8
+            recv_data = output.getInt8Data()
+            # Can optionally post-process the result
+            k = postprocess(recv_data, 5)
+            assert k == gold_response_output
     # -inference:
 
     print("custom_processing.py: Passed")
diff --git a/examples/python/hello_migraphx.py b/examples/python/hello_migraphx.py
index 8b7b3773f..0963984e1 100644
--- a/examples/python/hello_migraphx.py
+++ b/examples/python/hello_migraphx.py
@@ -71,7 +71,7 @@ def preprocess(img_data):
 
     Args:
         img_data (np.array): 3 dimensions [channels, rows, cols] with value range 0-255
-        the vectors are for RGB images, so images read with OpenCV must have channels 
+        the vectors are for RGB images, so images read with OpenCV must have channels
         converted before calling.
     """
     mean_vec = np.array([0.406, 0.456, 0.485])
@@ -287,7 +287,7 @@ def main(args):
     #
     # create a multi-image inference request and send it
     #
-    images = [img2, img, img, img2]
+    images = [img2]
 
     print("Creating inference request...")
     request = proteus.ImageInferenceRequest(images, False)
diff --git a/examples/python/hello_world_rest.py b/examples/python/hello_world_rest.py
index c0b1db2bb..ac8c3137f 100644
--- a/examples/python/hello_world_rest.py
+++ b/examples/python/hello_world_rest.py
@@ -21,12 +21,14 @@
 # fmt: off
 # +imports:
 from time import sleep
+
+import numpy as np
+
 import proteus
+
 # -imports:
 # fmt: on
 
-import numpy as np
-
 
 def NumericalInferenceRequest(data, datatype=proteus.DataType.UINT32):
     if not isinstance(data, list):
@@ -66,7 +68,7 @@ def main():
     # -load worker:
 
     # +inference: construct the request and make the inference
-    data = [3, 1, 4, 1, 5]
+    data = [3]
     request = NumericalInferenceRequest(data)
     response = client.modelInfer(worker_name, request)
     # -inference:
diff --git a/examples/python/migraph_resnet.py b/examples/python/migraph_resnet.py
index 28ebb572e..9f6aef588 100644
--- a/examples/python/migraph_resnet.py
+++ b/examples/python/migraph_resnet.py
@@ -28,10 +28,9 @@
 """
 
 
-import os
-
 import argparse
 import json  # json for reading labels file
+import os
 import sys
 import time
 
@@ -41,7 +40,6 @@
 import proteus
 import proteus.clients
 
-
 # The make_nxn and preprocess functions are based on an migraphx example at
 # AMDMIGraphx/examples/vision/python_resnet50/resnet50_inference.ipynb
 # The mean and standard dev. values used for this normalization are requirements of the Resnet50 model.
@@ -77,7 +75,7 @@ def preprocess(img_data):
 
     Args:
         img_data (np.array): 3 dimensions [channels, rows, cols] with value range 0-255
-        the vectors are for RGB images, so images read with OpenCV must have channels 
+        the vectors are for RGB images, so images read with OpenCV must have channels
         converted before calling.
     """
     mean_vec = np.array([0.406, 0.456, 0.485])
@@ -106,7 +104,7 @@ def parse_args():
         type=int,
         required=False,
         help="Number of images per REST request",
-        default=4,
+        default=1,
     )
 
     parser.add_argument(
@@ -192,7 +190,7 @@ def main(args):
     shape = []
     import onnx
 
-    print('Loading model to verify data shape...')
+    print("Loading model to verify data shape...")
     model = onnx.load(modelname)
     for input in model.graph.input:
         if input.name == "data":
@@ -253,7 +251,8 @@ def main(args):
     # list all the *.jpg images in the validation image directory
     files = os.listdir(validation_dir)
     files.sort()
-    files.remove("val.txt")
+    if validation_answers_file in files:
+        files.remove(validation_answers_file)
 
     # Read the "answers" file
     ground_truth = [None] * len(files)
@@ -284,47 +283,46 @@ def main(args):
             #  Normalize values with values specific to Resnet50
             imgV = imgV.transpose(2, 0, 1)
             imgV = preprocess(imgV)
-            images[index % request_size] = imgV
+            # images[index % request_size] = imgV
             index = index + 1
-            if index % request_size == 0:
-                print("processed so far: ", index, " of ", len(files))
-                print("Creating inference request with ", len(images), " items")
-                request = proteus.ImageInferenceRequest(images, False)
-                print("request is ready.  Sending...")
-                response = client.modelInfer(worker_name, request)
-                assert not response.isError(), response.getError()
-                print("Client received inference reply.")
-                j = 0
-                for output in response.getOutputs():
-                    assert output.datatype == proteus.DataType.FP32
-                    recv_data = output.getFp32Data()
-                    # the predicted category is the one with the highest match value
-                    answer = np.argmax(recv_data)
-                    step_index = index - request_size + j + 1
-                    print(
-                        "Reading result: best match category is ",
-                        answer,
-                        "  value is ",
-                        recv_data[answer],
-                        ".  This is a picture of a ",
-                        labels[answer],
-                        "    ground truth: ",
-                        ground_truth[step_index],
-                    )
-                    if int(answer) == int(ground_truth[step_index]):
-                        correct_answers = correct_answers + 1.0
-                    else:
-                        wrong_answers = wrong_answers + 1.0
-                    j = j + 1
+            print("processed so far: ", index, " of ", len(files))
+            print("Creating inference request with ", len(images), " items")
+            request = proteus.ImageInferenceRequest(imgV, False)
+            print("request is ready.  Sending...")
+            response = client.modelInfer(worker_name, request)
+            assert not response.isError(), response.getError()
+            print("Client received inference reply.")
+            j = 0
+            for output in response.getOutputs():
+                assert output.datatype == proteus.DataType.FP32
+                recv_data = output.getFp32Data()
+                # the predicted category is the one with the highest match value
+                answer = np.argmax(recv_data)
+                step_index = index - request_size + j + 1
                 print(
-                    "Correct: ",
-                    correct_answers,
-                    "  Wrong: ",
-                    wrong_answers,
-                    "    Accuracy: ",
-                    correct_answers / (correct_answers + wrong_answers),
+                    "Reading result: best match category is ",
+                    answer,
+                    "  value is ",
+                    recv_data[answer],
+                    ".  This is a picture of a ",
+                    labels[answer],
+                    "    ground truth: ",
+                    ground_truth[step_index],
                 )
-                print("     ----------------------------------------")
+                if int(answer) == int(ground_truth[step_index]):
+                    correct_answers = correct_answers + 1.0
+                else:
+                    wrong_answers = wrong_answers + 1.0
+                j = j + 1
+            print(
+                "Correct: ",
+                correct_answers,
+                "  Wrong: ",
+                wrong_answers,
+                "    Accuracy: ",
+                correct_answers / (correct_answers + wrong_answers),
+            )
+            print("     ----------------------------------------")
 
     print("Done")
 
diff --git a/examples/python/pt_zendnn.py b/examples/python/pt_zendnn.py
index 128c90829..03c47fdf3 100644
--- a/examples/python/pt_zendnn.py
+++ b/examples/python/pt_zendnn.py
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import argparse
+import functools
+import multiprocessing as mp
 import os
 import sys
 import time
@@ -23,6 +25,19 @@
 import proteus
 
 
+def infer(client, worker_name, image):
+    """
+    Make an inference. For multiprocessing, the function must be defined at the
+    top level
+
+    Args:
+        image (np.array): Image to send to the server
+    """
+    request = proteus.ImageInferenceRequest(image)
+    response = client.modelInfer(worker_name, request)
+    assert not response.isError(), response.getError()
+
+
 def main(args):
     """
     This is the main method which
@@ -127,6 +142,7 @@ def main(args):
         )
 
         # Run the inference for the images
+        processes = int(mp.cpu_count() / 2)
         while num_remaining_images >= batch_size:
 
             # create some random data
@@ -136,12 +152,12 @@ def main(args):
             images = [image for image in images]
 
             # Send request to the server
-            request = proteus.ImageInferenceRequest(images)
+            make_inference = functools.partial(infer, client, worker_name)
             start = time.time()
-            response = client.modelInfer(worker_name, request)
+            with mp.Pool(processes) as p:
+                p.map(make_inference, images)
             end = time.time()
             total_time += end - start
-            assert not response.isError(), response.getError()
 
             num_processed_images += batch_size
             num_remaining_images -= batch_size
diff --git a/examples/python/tf_zendnn.py b/examples/python/tf_zendnn.py
index 0c0577f0a..f86ef2f29 100644
--- a/examples/python/tf_zendnn.py
+++ b/examples/python/tf_zendnn.py
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import argparse
+import functools
+import multiprocessing as mp
 import os
 import sys
 import time
@@ -24,6 +26,19 @@
 import proteus.clients
 
 
+def infer(client, worker_name, image):
+    """
+    Make an inference. For multiprocessing, the function must be defined at the
+    top level
+
+    Args:
+        image (np.array): Image to send to the server
+    """
+    request = proteus.ImageInferenceRequest(image)
+    response = client.modelInfer(worker_name, request)
+    assert not response.isError(), response.getError()
+
+
 def main(args):
     """
     This is the main method which
@@ -133,6 +148,7 @@ def main(args):
         )
 
         # Run the inference for the images
+        processes = int(mp.cpu_count() / 2)
         while num_remaining_images >= batch_size:
 
             # create some random data
@@ -142,12 +158,12 @@ def main(args):
             images = [image for image in images]
 
             # Send request to the server
-            request = proteus.ImageInferenceRequest(images)
+            make_inference = functools.partial(infer, client, worker_name)
             start = time.time()
-            response = client.modelInfer(worker_name, request)
+            with mp.Pool(processes) as p:
+                p.map(make_inference, images)
             end = time.time()
             total_time += end - start
-            assert not response.isError(), response.getError()
 
             num_processed_images += batch_size
             num_remaining_images -= batch_size
diff --git a/include/proteus/clients/client.hpp b/include/proteus/clients/client.hpp
index 8c12eea65..ef7e11ee1 100644
--- a/include/proteus/clients/client.hpp
+++ b/include/proteus/clients/client.hpp
@@ -40,8 +40,11 @@ class Client {
   virtual void modelLoad(const std::string& model,
                          RequestParameters* parameters) = 0;
   virtual void modelUnload(const std::string& model) = 0;
+
   virtual InferenceResponse modelInfer(const std::string& model,
                                        const InferenceRequest& request) = 0;
+  virtual InferenceResponseFuture modelInferAsync(
+    const std::string& model, const InferenceRequest& request) = 0;
   virtual std::vector<std::string> modelList() = 0;
 
   virtual std::string workerLoad(const std::string& worker,
diff --git a/include/proteus/clients/grpc.hpp b/include/proteus/clients/grpc.hpp
index fa5f914ad..133de9c4c 100644
--- a/include/proteus/clients/grpc.hpp
+++ b/include/proteus/clients/grpc.hpp
@@ -24,7 +24,7 @@
 #include <string>  // for string
 #include <vector>  // for vector
 
-#include "proteus/clients/client.hpp"    // for Client
+#include "proteus/clients/client.hpp"    // IWYU pragma: export
 #include "proteus/core/predict_api.hpp"  // for InferenceRequest (ptr only)
 
 namespace grpc {
@@ -35,10 +35,9 @@ namespace proteus {
 
 class GrpcClient : public Client {
  public:
-  GrpcClient() = delete;
-  GrpcClient(const std::string& address);
-  GrpcClient(const std::shared_ptr<::grpc::Channel>& channel);
-  ~GrpcClient();
+  explicit GrpcClient(const std::string& address);
+  explicit GrpcClient(const std::shared_ptr<::grpc::Channel>& channel);
+  ~GrpcClient() override;
 
   ServerMetadata serverMetadata() override;
   bool serverLive() override;
@@ -49,8 +48,11 @@ class GrpcClient : public Client {
   void modelLoad(const std::string& model,
                  RequestParameters* parameters) override;
   void modelUnload(const std::string& model) override;
+
   InferenceResponse modelInfer(const std::string& model,
                                const InferenceRequest& request) override;
+  InferenceResponseFuture modelInferAsync(
+    const std::string& model, const InferenceRequest& request) override;
   std::vector<std::string> modelList() override;
   // int streamModelInferStart(const std::string& model,
   //                           const InferenceRequest& request,
diff --git a/include/proteus/clients/http.hpp b/include/proteus/clients/http.hpp
index 46e263d3a..458d90225 100644
--- a/include/proteus/clients/http.hpp
+++ b/include/proteus/clients/http.hpp
@@ -20,13 +20,13 @@
 #ifndef GUARD_PROTEUS_CLIENTS_HTTP
 #define GUARD_PROTEUS_CLIENTS_HTTP
 
-#include <memory>         // for unique_ptr
-#include <string>         // for string
-#include <unordered_map>  // for unordered_map
-#include <vector>         // for vector
+#include <memory>  // for unique_ptr
+#include <string>  // for string
+#include <vector>  // for vector
 
-#include "proteus/clients/client.hpp"    // for Client
-#include "proteus/core/predict_api.hpp"  // for InferenceRequest (ptr only)
+#include "proteus/clients/client.hpp"        // IWYU pragma: export
+#include "proteus/core/predict_api.hpp"      // for RequestParameters (ptr o...
+#include "proteus/helpers/declarations.hpp"  // for StringMap
 
 namespace proteus {
 
@@ -47,9 +47,12 @@ namespace proteus {
 
 class HttpClient : public Client {
  public:
-  HttpClient() = delete;
-  HttpClient(const std::string& address, const StringMap& headers = {});
-  ~HttpClient();
+  HttpClient(std::string address, const StringMap& headers = {});
+  HttpClient(HttpClient const&);
+  HttpClient& operator=(const HttpClient&) = delete;
+  HttpClient(HttpClient&& other) noexcept;
+  HttpClient& operator=(HttpClient&& other) noexcept = delete;
+  ~HttpClient() override;
 
   ServerMetadata serverMetadata() override;
   bool serverLive() override;
@@ -60,8 +63,11 @@ class HttpClient : public Client {
   void modelLoad(const std::string& model,
                  RequestParameters* parameters) override;
   void modelUnload(const std::string& model) override;
+
   InferenceResponse modelInfer(const std::string& model,
                                const InferenceRequest& request) override;
+  InferenceResponseFuture modelInferAsync(
+    const std::string& model, const InferenceRequest& request) override;
   std::vector<std::string> modelList() override;
 
   std::string workerLoad(const std::string& worker,
@@ -70,7 +76,15 @@ class HttpClient : public Client {
 
   bool hasHardware(const std::string& name, int num) override;
 
+  const std::string& getAddress() const&;
+  std::string getAddress() const&&;
+  const StringMap& getHeaders() const&;
+  StringMap getHeaders() const&&;
+
  private:
+  std::string address_;
+  StringMap headers_;
+
   class HttpClientImpl;
   std::unique_ptr<HttpClientImpl> impl_;
 };
diff --git a/include/proteus/clients/native.hpp b/include/proteus/clients/native.hpp
index 791562fc6..0d6f7bf31 100644
--- a/include/proteus/clients/native.hpp
+++ b/include/proteus/clients/native.hpp
@@ -20,11 +20,10 @@
 #ifndef GUARD_PROTEUS_CLIENTS_NATIVE
 #define GUARD_PROTEUS_CLIENTS_NATIVE
 
-#include <cstddef>  // for size_t
-#include <string>   // for string
-#include <vector>   // for vector
+#include <string>  // for string
+#include <vector>  // for vector
 
-#include "proteus/clients/client.hpp"        // for Client
+#include "proteus/clients/client.hpp"        // IWYU pragma: export
 #include "proteus/core/predict_api.hpp"      // for InferenceRequest (ptr only)
 #include "proteus/helpers/declarations.hpp"  // for InferenceResponseFuture
 
@@ -63,16 +62,9 @@ class NativeClient : public Client {
    * @param worker name of the worker to unload
    */
   void modelUnload(const std::string& model) override;
+
   InferenceResponse modelInfer(const std::string& model,
                                const InferenceRequest& request) override;
-  std::vector<std::string> modelList() override;
-
-  std::string workerLoad(const std::string& worker,
-                         RequestParameters* parameters) override;
-  void workerUnload(const std::string& worker) override;
-
-  bool hasHardware(const std::string& name, int num) override;
-
   /**
    * @brief Enqueue an inference request to Proteus
    *
@@ -80,8 +72,15 @@ class NativeClient : public Client {
    * @param request the request to make
    * @return InferenceResponseFuture a future to get the results of the request
    */
-  static InferenceResponseFuture enqueue(const std::string& workerName,
-                                         InferenceRequest request);
+  InferenceResponseFuture modelInferAsync(
+    const std::string& workerName, const InferenceRequest& request) override;
+  std::vector<std::string> modelList() override;
+
+  std::string workerLoad(const std::string& worker,
+                         RequestParameters* parameters) override;
+  void workerUnload(const std::string& worker) override;
+
+  bool hasHardware(const std::string& name, int num) override;
 };
 
 }  // namespace proteus
diff --git a/include/proteus/clients/websocket.hpp b/include/proteus/clients/websocket.hpp
index 30e8b4fdc..361d40deb 100644
--- a/include/proteus/clients/websocket.hpp
+++ b/include/proteus/clients/websocket.hpp
@@ -24,7 +24,7 @@
 #include <string>  // for string
 #include <vector>  // for vector
 
-#include "proteus/clients/client.hpp"    // for Client
+#include "proteus/clients/client.hpp"    // IWYU pragma: export
 #include "proteus/core/predict_api.hpp"  // for InferenceRequest (ptr only)
 
 namespace proteus {
@@ -47,6 +47,8 @@ class WebSocketClient : public Client {
   void modelUnload(const std::string& model) override;
   InferenceResponse modelInfer(const std::string& model,
                                const InferenceRequest& request) override;
+  InferenceResponseFuture modelInferAsync(
+    const std::string& model, const InferenceRequest& request) override;
   std::vector<std::string> modelList() override;
 
   std::string workerLoad(const std::string& worker,
@@ -55,8 +57,7 @@ class WebSocketClient : public Client {
 
   bool hasHardware(const std::string& name, int num) override;
 
-  void modelInferAsync(const std::string& model,
-                       const InferenceRequest& request);
+  void modelInferWs(const std::string& model, const InferenceRequest& request);
   // TODO(varunsh): change to InferenceReponse
   std::string modelRecv();
   void close();
diff --git a/include/proteus/core/data_types.hpp b/include/proteus/core/data_types.hpp
index aa03feeaf..a88d8f317 100644
--- a/include/proteus/core/data_types.hpp
+++ b/include/proteus/core/data_types.hpp
@@ -20,11 +20,10 @@
 #ifndef GUARD_PROTEUS_CORE_DATA_TYPES
 #define GUARD_PROTEUS_CORE_DATA_TYPES
 
-#include <cstddef>    // for size_t
-#include <cstdint>    // for uint8_t, int16_t, int32_t, int64_t
-#include <iostream>   // for ostream
-#include <stdexcept>  // for invalid_argument
-#include <string>     // for string
+#include <cstddef>   // for size_t
+#include <cstdint>   // for uint8_t, int16_t, int32_t
+#include <iostream>  // for ostream
+#include <string>    // for string
 
 #include "proteus/build_options.hpp"    // for PROTEUS_ENABLE_VITIS
 #include "proteus/core/exceptions.hpp"  // for invalid_argument
diff --git a/include/proteus/helpers/declarations.hpp b/include/proteus/helpers/declarations.hpp
index 128c6c970..3502a6569 100644
--- a/include/proteus/helpers/declarations.hpp
+++ b/include/proteus/helpers/declarations.hpp
@@ -25,7 +25,7 @@ namespace proteus {
 
 class Buffer;
 
-struct Batch;
+class Batch;
 
 class Interface;
 using InterfacePtr = std::unique_ptr<Interface>;
diff --git a/include/proteus/servers/server.hpp b/include/proteus/servers/server.hpp
index 56a4a3b4c..bf565d553 100644
--- a/include/proteus/servers/server.hpp
+++ b/include/proteus/servers/server.hpp
@@ -17,7 +17,6 @@
 
 #include <cstdint>
 #include <memory>
-#include <string>
 
 namespace proteus {
 
diff --git a/src/proteus/batching/batcher.cpp b/src/proteus/batching/batcher.cpp
index ba172ed2d..799c8a3f3 100644
--- a/src/proteus/batching/batcher.cpp
+++ b/src/proteus/batching/batcher.cpp
@@ -19,13 +19,16 @@
 
 #include "proteus/batching/batcher.hpp"
 
-#include <memory>   // for shared_ptr, make_sh...
+#include <cassert>  // for assert
+#include <memory>   // for shared_ptr, make_shared
 #include <string>   // for string
 #include <utility>  // for move
 
-#include "proteus/core/interface.hpp"
-#include "proteus/core/predict_api_internal.hpp"  // for RequestParameters
-#include "proteus/observation/logging.hpp"        // for getLogger, LoggerPtr
+#include "proteus/buffers/buffer.hpp"       // IWYU pragma: keep
+#include "proteus/core/interface.hpp"       // IWYU pragma: keep
+#include "proteus/core/worker_info.hpp"     // for WorkerInfo
+#include "proteus/observation/logging.hpp"  // for Logger, Loggers, Loggers:...
+
 namespace proteus {
 
 /**
@@ -37,20 +40,19 @@ namespace proteus {
 Batcher::Batcher() {
   this->input_queue_ = std::make_shared<BlockingQueue<InterfacePtr>>();
   this->output_queue_ = std::make_shared<BatchPtrQueue>();
-  this->batch_size_ = 1;
   this->status_ = BatcherStatus::kNew;
 #ifdef PROTEUS_ENABLE_LOGGING
   this->logger_ = Logger(Loggers::kServer);
 #endif
 }
 
-Batcher::Batcher(RequestParameters *parameters) : Batcher() {
+Batcher::Batcher(RequestParameters* parameters) : Batcher() {
   if (parameters != nullptr) {
     this->parameters_ = *parameters;
   }
 }
 
-Batcher::Batcher(const Batcher &batcher) {
+Batcher::Batcher(const Batcher& batcher) {
   this->input_queue_ = batcher.input_queue_;
   this->output_queue_ = batcher.output_queue_;
   this->batch_size_ = batcher.batch_size_;
@@ -61,7 +63,7 @@ Batcher::Batcher(const Batcher &batcher) {
   this->model_ = batcher.model_;
 }
 
-void Batcher::start(WorkerInfo *worker) {
+void Batcher::start(WorkerInfo* worker) {
   this->status_ = BatcherStatus::kRun;
   this->thread_ = std::thread(&Batcher::run, this, worker);
 }
@@ -70,22 +72,21 @@ void Batcher::setBatchSize(size_t batch_size) {
   this->batch_size_ = batch_size;
 }
 
-void Batcher::setName(const std::string &name) { this->model_ = name; }
+void Batcher::setName(const std::string& name) { this->model_ = name; }
 
 std::string Batcher::getName() { return this->model_; }
 
-BlockingQueue<InterfacePtr> *Batcher::getInputQueue() {
+BlockingQueue<InterfacePtr>* Batcher::getInputQueue() {
   return this->input_queue_.get();
 }
 
-BatchPtrQueue *Batcher::getOutputQueue() { return this->output_queue_.get(); }
+BatchPtrQueue* Batcher::getOutputQueue() { return this->output_queue_.get(); }
 
 void Batcher::enqueue(InterfacePtr request) {
   this->input_queue_->enqueue(std::move(request));
-  this->cv_.notify_one();
 }
 
-void Batcher::run(WorkerInfo *worker) {
+void Batcher::run(WorkerInfo* worker) {
   this->doRun(worker);
   this->status_ = BatcherStatus::kInactive;
 }
@@ -98,7 +99,84 @@ void Batcher::end() {
 }
 
 #ifdef PROTEUS_ENABLE_LOGGING
-const Logger &Batcher::getLogger() const { return logger_; }
+const Logger& Batcher::getLogger() const { return logger_; }
+#endif
+
+Batch::Batch(const WorkerInfo* worker)
+  : worker_(worker),
+    input_buffers_(worker->getInputBuffer()),
+    output_buffers_(worker->getOutputBuffer()) {}
+
+Batch::~Batch() {
+  worker_->putInputBuffer(std::move(input_buffers_));
+  worker_->putOutputBuffer(std::move(output_buffers_));
+}
+
+void Batch::addRequest(InferenceRequestPtr request) {
+  requests_.push_back(std::move(request));
+}
+
+std::vector<Buffer*> Batch::getRawInputBuffers() const {
+  std::vector<Buffer*> buffers;
+  buffers.reserve(input_buffers_.size());
+  for (const auto& buffer : input_buffers_) {
+    buffers.push_back(buffer.get());
+  }
+  return buffers;
+}
+
+const BufferPtrs& Batch::getInputBuffers() const { return input_buffers_; }
+
+std::vector<Buffer*> Batch::getRawOutputBuffers() const {
+  std::vector<Buffer*> buffers;
+  buffers.reserve(output_buffers_.size());
+  for (const auto& buffer : output_buffers_) {
+    buffers.push_back(buffer.get());
+  }
+  return buffers;
+}
+
+const BufferPtrs& Batch::getOutputBuffers() const { return output_buffers_; }
+
+const std::vector<InferenceRequestPtr>& Batch::getRequests() {
+  return requests_;
+}
+
+const InferenceRequestPtr& Batch::getRequest(int index) {
+  return requests_.at(index);
+}
+
+bool Batch::empty() const { return requests_.empty(); }
+
+size_t Batch::size() const {
+#ifdef PROTEUS_ENABLE_TRACING
+  assert(requests_.size() == traces_.size());
+#endif
+#ifdef PROTEUS_ENABLE_METRICS
+  assert(requests_.size() == start_times_.size());
+#endif
+
+  return requests_.size();
+}
+
+size_t Batch::input_size() const { return input_buffers_.size(); }
+
+size_t Batch::output_size() const { return output_buffers_.size(); }
+
+#ifdef PROTEUS_ENABLE_TRACING
+void Batch::addTrace(TracePtr trace) { traces_.push_back(std::move(trace)); }
+
+TracePtr& Batch::getTrace(int index) { return traces_.at(index); }
+#endif
+
+#ifdef PROTEUS_ENABLE_METRICS
+void Batch::addTime(std::chrono::high_resolution_clock::time_point timestamp) {
+  start_times_.push_back(std::move(timestamp));
+}
+
+std::chrono::high_resolution_clock::time_point Batch::getTime(int index) {
+  return start_times_.at(index);
+}
 #endif
 
 }  // namespace proteus
diff --git a/src/proteus/batching/batcher.hpp b/src/proteus/batching/batcher.hpp
index b14509e4a..cb5e65ca2 100644
--- a/src/proteus/batching/batcher.hpp
+++ b/src/proteus/batching/batcher.hpp
@@ -20,14 +20,12 @@
 #ifndef GUARD_PROTEUS_BATCHING_BATCHER
 #define GUARD_PROTEUS_BATCHING_BATCHER
 
-#include <chrono>              // for system_clock::time_point
-#include <condition_variable>  // for condition_variable
-#include <cstddef>             // for size_t
-#include <memory>              // for unique_ptr, shared_ptr
-#include <mutex>               // for mutex
-#include <string>              // for string
-#include <thread>              // for thread
-#include <vector>              // for vector
+#include <chrono>   // for system_clock::time_point
+#include <cstddef>  // for size_t
+#include <memory>   // for unique_ptr, shared_ptr
+#include <string>   // for string
+#include <thread>   // for thread
+#include <vector>   // for vector
 
 #include "proteus/build_options.hpp"         // for PROTEUS_ENABLE_LOGGING
 #include "proteus/core/predict_api.hpp"      // for RequestParameters
@@ -37,6 +35,7 @@
 #include "proteus/observation/tracing.hpp"   // for TracePtr
 
 namespace proteus {
+class Buffer;
 class WorkerInfo;
 }  // namespace proteus
 
@@ -50,15 +49,47 @@ enum class BatcherStatus { kNew, kRun, kInactive, kDead };
  * metadata that should be sent to the worker.
  *
  */
-struct Batch {
-  std::unique_ptr<std::vector<InferenceRequestPtr>> requests;
-  std::unique_ptr<std::vector<BufferPtrs>> input_buffers;
-  std::unique_ptr<std::vector<BufferPtrs>> output_buffers;
+class Batch {
+ public:
+  explicit Batch(const WorkerInfo* worker);
+  ~Batch();
+
+  void addRequest(InferenceRequestPtr request);
+
+  const InferenceRequestPtr& getRequest(int index);
+  const std::vector<InferenceRequestPtr>& getRequests();
+  const BufferPtrs& getInputBuffers() const;
+  const BufferPtrs& getOutputBuffers() const;
+  std::vector<Buffer*> getRawInputBuffers() const;
+  std::vector<Buffer*> getRawOutputBuffers() const;
+
+  bool empty() const;
+  size_t size() const;
+  size_t input_size() const;
+  size_t output_size() const;
+
+#ifdef PROTEUS_ENABLE_TRACING
+  void addTrace(TracePtr trace);
+  TracePtr& getTrace(int index);
+#endif
+#ifdef PROTEUS_ENABLE_METRICS
+  void addTime(std::chrono::high_resolution_clock::time_point timestamp);
+  std::chrono::high_resolution_clock::time_point getTime(int index);
+#endif
+
+  auto begin() const { return requests_.begin(); }
+  auto end() const { return requests_.end(); }
+
+ private:
+  const WorkerInfo* worker_;
+  std::vector<InferenceRequestPtr> requests_;
+  std::vector<BufferPtr> input_buffers_;
+  std::vector<BufferPtr> output_buffers_;
 #ifdef PROTEUS_ENABLE_TRACING
-  std::vector<TracePtr> traces;
+  std::vector<TracePtr> traces_;
 #endif
 #ifdef PROTEUS_ENABLE_METRICS
-  std::vector<std::chrono::high_resolution_clock::time_point> start_times;
+  std::vector<std::chrono::high_resolution_clock::time_point> start_times_;
 #endif
 };
 
@@ -139,8 +170,6 @@ class Batcher {
   std::shared_ptr<BlockingQueue<InterfacePtr>> input_queue_;
   std::shared_ptr<BatchPtrQueue> output_queue_;
   std::thread thread_;
-  std::condition_variable cv_;
-  std::mutex cv_m_;
   std::string model_;
   RequestParameters parameters_;
 
diff --git a/src/proteus/batching/hard.cpp b/src/proteus/batching/hard.cpp
index 5c4bfd16e..c9afe9819 100644
--- a/src/proteus/batching/hard.cpp
+++ b/src/proteus/batching/hard.cpp
@@ -19,29 +19,21 @@
 
 #include "proteus/batching/hard.hpp"
 
-#include <chrono>     // for system_clock::time_point
-#include <cstddef>    // for size_t
-#include <memory>     // for unique_ptr, make_unique
-#include <mutex>      // for mutex, unique_lock
-#include <stdexcept>  // for invalid_argument
-#include <string>     // for operator+
-#include <utility>    // for move
-#include <vector>     // for vector
-
-#include "proteus/buffers/buffer.hpp"        // IWYU pragma: keep
-#include "proteus/build_options.hpp"         // for PROTEUS_ENABLE_TRACING
-#include "proteus/core/interface.hpp"        // for InterfacePtr, Interface
-#include "proteus/core/manager.hpp"          // for Manager
-#include "proteus/core/worker_info.hpp"      // for WorkerInfo
-#include "proteus/helpers/declarations.hpp"  // for BufferPtr, InferenceRequ...
+#include <cstddef>  // for size_t
+#include <memory>   // for unique_ptr, operator==
+#include <string>   // for operator+
+#include <utility>  // for move
+#include <vector>   // for vector
+
+#include "proteus/build_options.hpp"         // for PROTEUS_ENABLE_METRICS
+#include "proteus/core/exceptions.hpp"       // for invalid_argument
+#include "proteus/core/interface.hpp"        // for Interface
+#include "proteus/helpers/declarations.hpp"  // for InterfacePtr
 #include "proteus/helpers/queue.hpp"         // for BlockingConcurrentQueue
 #include "proteus/helpers/thread.hpp"        // for setThreadName
-#include "proteus/observation/metrics.hpp"   // for Metrics
-#include "proteus/observation/tracing.hpp"   // for startFollowSpan, SpanPtr
+#include "proteus/observation/metrics.hpp"   // for Metrics, MetricCounterIDs
+#include "proteus/observation/tracing.hpp"   // for Trace
 
-namespace proteus {
-class InferenceRequest;
-}  // namespace proteus
 // IWYU pragma: no_forward_declare proteus::Buffer
 
 namespace proteus {
@@ -49,33 +41,18 @@ namespace proteus {
 void HardBatcher::doRun(WorkerInfo* worker) {
   auto thread_name = "batch" + this->getName();
   setThreadName(thread_name);
-  std::unique_lock<std::mutex> lk(this->cv_m_);
   InterfacePtr req;
   bool run = true;
 
   while (run) {
-    auto batch = std::make_unique<Batch>();
-    batch->requests =
-      std::make_unique<std::vector<std::shared_ptr<InferenceRequest>>>();
-    batch->input_buffers = std::make_unique<std::vector<BufferPtrs>>();
-    batch->output_buffers = std::make_unique<std::vector<BufferPtrs>>();
-    auto input_buffer = worker->getInputBuffer();
-    std::vector<size_t> input_offset = {0};
-    auto output_buffer = worker->getOutputBuffer();
-    std::vector<size_t> output_offset = {0};
+    auto batch = std::make_unique<Batch>(worker);
+    auto input_buffers = batch->getRawInputBuffers();
+    auto output_buffers = batch->getRawOutputBuffers();
+    std::vector<size_t> input_offset(input_buffers.size(), 0);
+    std::vector<size_t> output_offset(output_buffers.size(), 0);
     size_t batch_size = 0;
 
     do {
-      std::vector<BufferRawPtrs> input_buffers;
-      input_buffers.emplace_back();
-      for (const auto& buffer : input_buffer) {
-        input_buffers.back().push_back(buffer.get());
-      }
-      std::vector<BufferRawPtrs> output_buffers;
-      output_buffers.emplace_back();
-      for (const auto& buffer : output_buffer) {
-        output_buffers.back().push_back(buffer.get());
-      }
       this->input_queue_->wait_dequeue(req);
 
       if (req == nullptr) {
@@ -93,13 +70,10 @@ void HardBatcher::doRun(WorkerInfo* worker) {
         MetricCounterIDs::kPipelineIngressBatcher);
 #endif
 
-      size_t input_size = 0;
-      try {
-        input_size = req->getInputSize();
-        if (!worker->inputSizeValid(input_size)) {
-          Manager::getInstance().workerAllocate(this->model_, input_size);
-        }
-      } catch (const invalid_argument& e) {
+      auto input_size = req->getInputSize();
+      if (input_size != input_buffers.size()) {
+        auto e =
+          invalid_argument("Number of input tensors do not match worker");
         req->errorHandler(e);
         continue;
       }
@@ -108,74 +82,33 @@ void HardBatcher::doRun(WorkerInfo* worker) {
         req->errorHandler(error);
         continue;
       }
-      auto buffers_needed =
-        ((input_size + batch_size - 1) / this->batch_size_) + 1;
-      if (buffers_needed > 1) {
-        for (size_t i = 1; i < buffers_needed; i++) {
-          batch->input_buffers->push_back(std::move(input_buffer));
-          batch->output_buffers->push_back(std::move(output_buffer));
-          input_buffer = worker->getInputBuffer();
-          // input_buffers.push_back(input_buffer.get());
-          input_buffers.emplace_back();
-          for (const auto& buffer : input_buffer) {
-            input_buffers.back().push_back(buffer.get());
-          }
-          output_buffer = worker->getOutputBuffer();
-          // output_buffers.push_back(output_buffer.get());
-          output_buffers.emplace_back();
-          for (const auto& buffer : output_buffer) {
-            output_buffers.back().push_back(buffer.get());
-          }
-          input_offset.push_back(0);
-          output_offset.push_back(0);
-        }
-      }
 
       auto old_input_offset = input_offset;
       auto old_output_offset = output_offset;
-      size_t buffer_index = 0;
-      auto new_req = req->getRequest(buffer_index, input_buffers, input_offset,
-                                     output_buffers, output_offset,
-                                     this->batch_size_, batch_size);
+      auto new_req = req->getRequest(input_buffers, input_offset,
+                                     output_buffers, output_offset);
       if (new_req == nullptr) {
-        // if the request fails, revert back the state
-        for (size_t i = 1; i < buffers_needed; i++) {
-          worker->putInputBuffer(std::move(input_buffer));
-          input_buffer = std::move(batch->input_buffers->back());
-          batch->input_buffers->pop_back();
-
-          worker->putOutputBuffer(std::move(output_buffer));
-          output_buffer = std::move(batch->output_buffers->back());
-          batch->output_buffers->pop_back();
-
-          input_buffers.pop_back();
-          output_buffers.pop_back();
-        }
         input_offset = old_input_offset;
         output_offset = old_output_offset;
       } else {
-        batch->requests->push_back(new_req);
+        batch->addRequest(new_req);
+        batch_size++;
 #ifdef PROTEUS_ENABLE_TRACING
         trace->endSpan();
-        batch->traces.emplace_back(std::move(trace));
+        batch->addTrace(std::move(trace));
 #endif
 #ifdef PROTEUS_ENABLE_METRICS
-        batch->start_times.emplace_back(req->get_time());
+        batch->addTime(req->get_time());
 #endif
       }
     } while (batch_size % this->batch_size_ != 0);
 
-    if (!batch->requests->empty()) {
-      batch->input_buffers->push_back(std::move(input_buffer));
-      batch->output_buffers->push_back(std::move(output_buffer));
+    if (!batch->empty()) {
       this->output_queue_->enqueue(std::move(batch));
 #ifdef PROTEUS_ENABLE_METRICS
       Metrics::getInstance().incrementCounter(
         MetricCounterIDs::kPipelineEgressBatcher);
 #endif
-    } else {
-      worker->putInputBuffer(std::move(input_buffer));
-      worker->putOutputBuffer(std::move(output_buffer));
     }
   }
 }
diff --git a/src/proteus/batching/soft.cpp b/src/proteus/batching/soft.cpp
index bb36f88d1..362e9427b 100644
--- a/src/proteus/batching/soft.cpp
+++ b/src/proteus/batching/soft.cpp
@@ -20,29 +20,25 @@
 #include "proteus/batching/soft.hpp"
 
 #include <algorithm>  // for max
-#include <chrono>     // for system_clock::time_point
+#include <chrono>     // for milliseconds, duration_cast
 #include <cstddef>    // for size_t
 #include <cstdint>    // for int32_t
 #include <memory>     // for unique_ptr, allocator
-#include <queue>      // for queue
 #include <ratio>      // for ratio
-#include <stdexcept>  // for invalid_argument
 #include <string>     // for operator+, char_traits
 #include <utility>    // for move
 #include <vector>     // for vector
 
-#include "proteus/buffers/buffer.hpp"        // IWYU pragma: keep
 #include "proteus/build_options.hpp"         // for PROTEUS_ENABLE_METRICS
+#include "proteus/core/exceptions.hpp"       // for invalid_argument
 #include "proteus/core/interface.hpp"        // for Interface
-#include "proteus/core/manager.hpp"          // for Manager
-#include "proteus/core/predict_api.hpp"      // for RequestParameters, Infer...
-#include "proteus/core/worker_info.hpp"      // for WorkerInfo
-#include "proteus/helpers/declarations.hpp"  // for InterfacePtr, BufferPtrs
+#include "proteus/core/predict_api.hpp"      // for RequestParameters
+#include "proteus/helpers/declarations.hpp"  // for InterfacePtr
 #include "proteus/helpers/queue.hpp"         // for BlockingConcurrentQueue
 #include "proteus/helpers/thread.hpp"        // for setThreadName
-#include "proteus/observation/logging.hpp"   // for Logger
+#include "proteus/observation/logging.hpp"   // for Logger, PROTEUS_LOG_DEBUG
 #include "proteus/observation/metrics.hpp"   // for Metrics, MetricCounterIDs
-#include "proteus/observation/tracing.hpp"   // for TracePtr, Trace
+#include "proteus/observation/tracing.hpp"   // for Trace
 
 using std::chrono::duration_cast;
 using std::chrono::milliseconds;
@@ -66,18 +62,12 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
       milliseconds(this->parameters_.get<int32_t>("timeout")));
   }
 
-  std::queue<InterfacePtr> backup;
-
   while (run) {
-    auto batch = std::make_unique<Batch>();
-    batch->requests =
-      std::make_unique<std::vector<std::shared_ptr<InferenceRequest>>>();
-    batch->input_buffers = std::make_unique<std::vector<BufferPtrs>>();
-    batch->output_buffers = std::make_unique<std::vector<BufferPtrs>>();
-    auto input_buffer = worker->getInputBuffer();
-    std::vector<size_t> input_offset = {0};
-    auto output_buffer = worker->getOutputBuffer();
-    std::vector<size_t> output_offset = {0};
+    auto batch = std::make_unique<Batch>(worker);
+    auto input_buffers = batch->getRawInputBuffers();
+    auto output_buffers = batch->getRawOutputBuffers();
+    std::vector<size_t> input_offset(input_buffers.size(), 0);
+    std::vector<size_t> output_offset(output_buffers.size(), 0);
     size_t batch_size = 0;
 
 #ifdef PROTEUS_ENABLE_METRICS
@@ -94,24 +84,19 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
 
     do {
       InterfacePtr req;
-      if (!backup.empty()) {
-        req = std::move(backup.front());
-        backup.pop();
+      if (first_request) {
+        // wait for the first request
+        this->input_queue_->wait_dequeue(req);
+        start_time = std::chrono::high_resolution_clock::now();
+        PROTEUS_LOG_DEBUG(logger,
+                          "Got request of a new batch for " + this->model_);
       } else {
-        if (first_request) {
-          // wait for the first request
-          this->input_queue_->wait_dequeue(req);
-          start_time = std::chrono::high_resolution_clock::now();
-          PROTEUS_LOG_DEBUG(logger,
-                            "Got request of a new batch for " + this->model_);
-        } else {
-          auto remaining_time =
-            kTimeout - (std::chrono::high_resolution_clock::now() - start_time);
-          auto duration = std::max(remaining_time, std::chrono::nanoseconds(0));
-          bool valid = this->input_queue_->wait_dequeue_timed(req, duration);
-          if (!valid) {
-            break;
-          }
+        auto remaining_time =
+          kTimeout - (std::chrono::high_resolution_clock::now() - start_time);
+        auto duration = std::max(remaining_time, std::chrono::nanoseconds(0));
+        bool valid = this->input_queue_->wait_dequeue_timed(req, duration);
+        if (!valid) {
+          break;
         }
       }
 
@@ -120,13 +105,10 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
         break;
       }
 
-      size_t input_size = 0;
-      try {
-        input_size = req->getInputSize();
-        if (!worker->inputSizeValid(input_size)) {
-          Manager::getInstance().workerAllocate(this->model_, input_size);
-        }
-      } catch (const invalid_argument& e) {
+      auto input_size = req->getInputSize();
+      if (input_size != input_buffers.size()) {
+        auto e =
+          invalid_argument("Number of input tensors do not match worker");
         req->errorHandler(e);
         continue;
       }
@@ -136,28 +118,6 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
         continue;
       }
 
-      if (input_size > batch_size_) {
-        auto error = invalid_argument("Input size is larger than batch size");
-        req->errorHandler(error);
-        continue;
-      }
-
-      if (batch_size + input_size > batch_size_) {
-        backup.push(std::move(req));
-        break;
-      }
-
-      std::vector<BufferRawPtrs> input_buffers;
-      input_buffers.emplace_back();
-      for (const auto& buffer : input_buffer) {
-        input_buffers.back().push_back(buffer.get());
-      }
-      std::vector<BufferRawPtrs> output_buffers;
-      output_buffers.emplace_back();
-      for (const auto& buffer : output_buffer) {
-        output_buffers.back().push_back(buffer.get());
-      }
-
 #ifdef PROTEUS_ENABLE_TRACING
       auto trace = req->getTrace();
       trace->startSpan("soft_batcher");
@@ -168,77 +128,38 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
         MetricCounterIDs::kPipelineIngressBatcher);
 #endif
 
-      auto buffers_needed =
-        ((input_size + batch_size - 1) / this->batch_size_) + 1;
-      if (buffers_needed > 1) {
-        for (size_t i = 1; i < buffers_needed; i++) {
-          batch->input_buffers->push_back(std::move(input_buffer));
-          batch->output_buffers->push_back(std::move(output_buffer));
-          input_buffer = worker->getInputBuffer();
-          input_buffers.emplace_back();
-          for (const auto& buffer : input_buffer) {
-            input_buffers.back().push_back(buffer.get());
-          }
-          output_buffer = worker->getOutputBuffer();
-          output_buffers.emplace_back();
-          for (const auto& buffer : output_buffer) {
-            output_buffers.back().push_back(buffer.get());
-          }
-          input_offset.push_back(0);
-          output_offset.push_back(0);
-        }
-      }
-
       auto old_input_offset = input_offset;
       auto old_output_offset = output_offset;
-      size_t buffer_index = 0;
-      auto new_req = req->getRequest(buffer_index, input_buffers, input_offset,
-                                     output_buffers, output_offset,
-                                     this->batch_size_, batch_size);
+      auto new_req = req->getRequest(input_buffers, input_offset,
+                                     output_buffers, output_offset);
       if (new_req == nullptr) {
         PROTEUS_LOG_DEBUG(logger, "Making request for " + this->model_ +
                                     " failed. Reverting buffers.");
-        for (size_t i = 1; i < buffers_needed; i++) {
-          worker->putInputBuffer(std::move(input_buffer));
-          input_buffer = std::move(batch->input_buffers->back());
-          batch->input_buffers->pop_back();
-
-          worker->putOutputBuffer(std::move(output_buffer));
-          output_buffer = std::move(batch->output_buffers->back());
-          batch->output_buffers->pop_back();
-
-          input_buffers.pop_back();
-          output_buffers.pop_back();
-        }
         input_offset = old_input_offset;
         output_offset = old_output_offset;
       } else {
-        batch->requests->push_back(new_req);
+        batch->addRequest(new_req);
+        batch_size++;
         if (first_request) {
           first_request = false;
         }
 #ifdef PROTEUS_ENABLE_TRACING
         trace->endSpan();
-        batch->traces.emplace_back(std::move(trace));
+        batch->addTrace(std::move(trace));
 #endif
 #ifdef PROTEUS_ENABLE_METRICS
-        batch->start_times.emplace_back(req->get_time());
+        batch->addTime(req->get_time());
 #endif
       }
     } while (batch_size % this->batch_size_ != 0 && run);
 
-    if (!batch->requests->empty()) {
+    if (!batch->empty()) {
       PROTEUS_LOG_DEBUG(logger, "Enqueuing batch for " + this->model_);
-      batch->input_buffers->push_back(std::move(input_buffer));
-      batch->output_buffers->push_back(std::move(output_buffer));
       this->output_queue_->enqueue(std::move(batch));
 #ifdef PROTEUS_ENABLE_METRICS
       Metrics::getInstance().incrementCounter(
         MetricCounterIDs::kPipelineEgressBatcher);
 #endif
-    } else {
-      worker->putInputBuffer(std::move(input_buffer));
-      worker->putOutputBuffer(std::move(output_buffer));
     }
   }
 }
diff --git a/src/proteus/bindings/python/clients/http.cpp b/src/proteus/bindings/python/clients/http.cpp
index 8d123909e..11f5af2d5 100644
--- a/src/proteus/bindings/python/clients/http.cpp
+++ b/src/proteus/bindings/python/clients/http.cpp
@@ -19,10 +19,14 @@
 
 #include "proteus/clients/http.hpp"
 
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include <pybind11/cast.h>      // for arg
+#include <pybind11/pybind11.h>  // for class_, init
+#include <pybind11/stl.h>       // IWYU pragma: keep
 
-#include "proteus/bindings/python/helpers/docstrings.hpp"
+#include <unordered_map>  // for unordered_map
+
+#include "proteus/bindings/python/helpers/docstrings.hpp"  // for DOCS
+#include "proteus/helpers/declarations.hpp"
 
 namespace py = pybind11;
 
@@ -55,5 +59,15 @@ void wrapHttpClient(py::module_ &m) {
          py::arg("request"), DOCS(HttpClient, modelInfer))
     .def("modelList", &HttpClient::modelList, DOCS(HttpClient, modelList))
     .def("hasHardware", &HttpClient::hasHardware, py::arg("name"),
-         py::arg("num"), DOCS(HttpClient, modelList));
+         py::arg("num"), DOCS(HttpClient, modelList))
+    .def(py::pickle(
+      [](const HttpClient &p) {  // __getstate__
+        return py::make_tuple(p.getAddress(), p.getHeaders());
+      },
+      [](py::tuple t) -> HttpClient {  // __setstate__
+        if (t.size() != 2) {
+          throw proteus::runtime_error("Invalid state!");
+        }
+        return {t[0].cast<std::string>(), t[1].cast<proteus::StringMap>()};
+      }));
 }
diff --git a/src/proteus/bindings/python/clients/websocket.cpp b/src/proteus/bindings/python/clients/websocket.cpp
index df5de8125..4906d906f 100644
--- a/src/proteus/bindings/python/clients/websocket.cpp
+++ b/src/proteus/bindings/python/clients/websocket.cpp
@@ -19,12 +19,15 @@
 
 #include "proteus/clients/websocket.hpp"
 
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include <pybind11/cast.h>      // for arg
+#include <pybind11/pybind11.h>  // for class_, init
+#include <pybind11/stl.h>       // IWYU pragma: keep
 
-#include <sstream>
+#include "proteus/bindings/python/helpers/docstrings.hpp"  // for DOCS
 
-#include "proteus/bindings/python/helpers/docstrings.hpp"
+namespace proteus {
+class DataType;
+}
 
 namespace py = pybind11;
 
@@ -52,8 +55,8 @@ void wrapWebSocketClient(py::module_ &m) {
          DOCS(WebSocketClient, modelUnload))
     .def("modelInfer", &WebSocketClient::modelInfer, py::arg("model"),
          py::arg("request"), DOCS(WebSocketClient, modelInfer))
-    .def("modelInferAsync", &WebSocketClient::modelInferAsync, py::arg("model"),
-         py::arg("request"), DOCS(WebSocketClient, modelInferAsync))
+    .def("modelInferWs", &WebSocketClient::modelInferWs, py::arg("model"),
+         py::arg("request"), DOCS(WebSocketClient, modelInferWs))
     .def("modelRecv", &WebSocketClient::modelRecv,
          DOCS(WebSocketClient, modelRecv))
     .def("modelList", &WebSocketClient::modelList,
diff --git a/src/proteus/bindings/python/core/data_types.cpp b/src/proteus/bindings/python/core/data_types.cpp
index 5f7a04dac..aa81b4019 100644
--- a/src/proteus/bindings/python/core/data_types.cpp
+++ b/src/proteus/bindings/python/core/data_types.cpp
@@ -19,17 +19,20 @@
 
 #include "proteus/core/data_types.hpp"
 
-#include <pybind11/operators.h>
-#include <pybind11/pybind11.h>
+#include <pybind11/cast.h>       // for arg
+#include <pybind11/operators.h>  // for self, operator==, self_t
+#include <pybind11/pybind11.h>   // for class_, enum_, object, init, module_
 
-#include <sstream>
+#include <sstream>  // IWYU pragma: keep
+
+#include "proteus/build_options.hpp"  // for PROTEUS_ENABLE_VITIS
+
+// IWYU pragma: no_forward_declare xir::Datatype
 
 #ifdef PROTEUS_ENABLE_VITIS
-#include <xir/util/data_type.hpp>
+#include <xir/util/data_type.hpp>  // IWYU pragma: keep
 #endif
 
-#include "proteus/bindings/python/helpers/docstrings.hpp"
-
 namespace py = pybind11;
 
 using proteus::DataType;
diff --git a/src/proteus/bindings/python/core/predict_api.cpp b/src/proteus/bindings/python/core/predict_api.cpp
index f27bf2e44..89cb5448b 100644
--- a/src/proteus/bindings/python/core/predict_api.cpp
+++ b/src/proteus/bindings/python/core/predict_api.cpp
@@ -19,14 +19,18 @@
 
 #include "proteus/core/predict_api.hpp"
 
-#include <pybind11/numpy.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include <pybind11/attr.h>      // for keep_alive
+#include <pybind11/cast.h>      // for arg
+#include <pybind11/numpy.h>     // for array_t
+#include <pybind11/pybind11.h>  // for class_, init
+#include <pybind11/stl.h>       // IWYU pragma: keep
 
-#include <sstream>
+#include <array>          // for array
+#include <sstream>        // IWYU pragma: keep
+#include <unordered_map>  // for unordered_map
 
-#include "proteus/bindings/python/helpers/docstrings.hpp"
-#include "proteus/bindings/python/helpers/print.hpp"
+#include "proteus/bindings/python/helpers/docstrings.hpp"  // for DOCS
+#include "proteus/bindings/python/helpers/print.hpp"       // for to_string
 
 namespace py = pybind11;
 
diff --git a/src/proteus/bindings/python/helpers/docstrings.hpp b/src/proteus/bindings/python/helpers/docstrings.hpp
index a1866fa84..cbba0e4ff 100644
--- a/src/proteus/bindings/python/helpers/docstrings.hpp
+++ b/src/proteus/bindings/python/helpers/docstrings.hpp
@@ -15,7 +15,7 @@
 #ifndef GUARD_PROTEUS_BINDINGS_PYTHON_HELPERS_DOCSTRINGS
 #define GUARD_PROTEUS_BINDINGS_PYTHON_HELPERS_DOCSTRINGS
 
-#include "auto_docstrings.hpp"
+#include "auto_docstrings.hpp"  // IWYU pragma: export
 
 #define DOCS(...) DOC(proteus, __VA_ARGS__)
 
diff --git a/src/proteus/bindings/python/proteus.cpp b/src/proteus/bindings/python/proteus.cpp
index 2e6a3b637..05bf82b12 100644
--- a/src/proteus/bindings/python/proteus.cpp
+++ b/src/proteus/bindings/python/proteus.cpp
@@ -23,7 +23,6 @@
 #include "proteus/bindings/python/core/data_types.hpp"
 #include "proteus/bindings/python/core/predict_api.hpp"
 #include "proteus/bindings/python/servers/server.hpp"
-#include "proteus/clients/native.hpp"
 #include "proteus/core/exceptions.hpp"
 
 namespace py = pybind11;
diff --git a/src/proteus/bindings/python/servers/server.cpp b/src/proteus/bindings/python/servers/server.cpp
index 6ee7103d0..dc2c6b3d0 100644
--- a/src/proteus/bindings/python/servers/server.cpp
+++ b/src/proteus/bindings/python/servers/server.cpp
@@ -19,8 +19,9 @@
 
 #include "proteus/servers/server.hpp"
 
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include <pybind11/cast.h>      // for arg
+#include <pybind11/pybind11.h>  // for class_, init
+#include <pybind11/stl.h>       // IWYU pragma: keep
 
 #include "proteus/bindings/python/helpers/docstrings.hpp"
 
diff --git a/src/proteus/buffers/buffer.hpp b/src/proteus/buffers/buffer.hpp
index ea56d773b..78122c15c 100644
--- a/src/proteus/buffers/buffer.hpp
+++ b/src/proteus/buffers/buffer.hpp
@@ -22,11 +22,8 @@
 #define GUARD_PROTEUS_BUFFERS_BUFFER
 
 #include <cstddef>  // for size_t
-#include <cstdint>  // for int16_t, int32_t, int64_t
-#include <string>
-
-#include "proteus/build_options.hpp"        // for PROTEUS_ENABLE_LOGGING
-#include "proteus/observation/logging.hpp"  // for getLogger, LoggerPtr
+#include <cstdint>  // for int16_t, int32_t, int64_t, int8_t, uint16_t, uint...
+#include <string>   // for string
 
 namespace proteus {
 
diff --git a/src/proteus/buffers/vector_buffer.cpp b/src/proteus/buffers/vector_buffer.cpp
index 9b9936d56..85523ecc8 100644
--- a/src/proteus/buffers/vector_buffer.cpp
+++ b/src/proteus/buffers/vector_buffer.cpp
@@ -19,8 +19,9 @@
 
 #include "proteus/buffers/vector_buffer.hpp"
 
-#include <memory>   // for make_unique
-#include <utility>  // for move
+#include <algorithm>  // for max
+#include <memory>     // for make_unique
+#include <utility>    // for move
 
 #include "proteus/helpers/declarations.hpp"  // for BufferPtrs
 
diff --git a/src/proteus/clients/client.cpp b/src/proteus/clients/client.cpp
index 628d3ddaf..63e6a1275 100644
--- a/src/proteus/clients/client.cpp
+++ b/src/proteus/clients/client.cpp
@@ -14,8 +14,8 @@
 
 #include "proteus/clients/client.hpp"
 
-#include "proteus/observation/logging.hpp"
-#include "proteus/servers/server.hpp"
+#include "proteus/build_options.hpp"        // for PROTEUS_ENABLE_LOGGING
+#include "proteus/observation/logging.hpp"  // for getLogDirectory, initLogger
 
 namespace proteus {
 
diff --git a/src/proteus/clients/grpc.cpp b/src/proteus/clients/grpc.cpp
index 8fa8c71d2..12e723a61 100644
--- a/src/proteus/clients/grpc.cpp
+++ b/src/proteus/clients/grpc.cpp
@@ -21,28 +21,26 @@
 
 #include <google/protobuf/repeated_ptr_field.h>  // for RepeatedPtrField
 #include <google/protobuf/stubs/common.h>        // for string
-#include <grpcpp/grpcpp.h>                       // for ClientContext, Creat...
+#include <grpcpp/grpcpp.h>                       // for Status, ClientContext
 
 #include <cstddef>        // for byte, size_t
 #include <cstdint>        // for uint64_t, uint32_t
 #include <cstring>        // for memcpy
+#include <future>         // for async
 #include <iostream>       // for operator<<, cout
 #include <map>            // for map
 #include <memory>         // for make_shared, reinter...
-#include <stdexcept>      // for runtime_error
-#include <string>         // for string, operator+
+#include <string>         // for string, basic_string
 #include <unordered_set>  // for unordered_set
 #include <utility>        // for move
 #include <variant>        // for visit
 #include <vector>         // for vector
 
 #include "predict_api.grpc.pb.h"             // for GRPCInferenceService...
-#include "predict_api.pb.h"                  // for InferTensorContents
-#include "proteus/build_options.hpp"         // for PROTEUS_ENABLE_GRPC
+#include "predict_api.pb.h"                  // for RepeatedField, Infer...
 #include "proteus/core/data_types.hpp"       // for DataType, DataType::...
 #include "proteus/core/exceptions.hpp"       // for bad_status
 #include "proteus/helpers/declarations.hpp"  // for InferenceResponseOutput
-#include "proteus/servers/grpc_server.hpp"   // for start, stop
 
 using grpc::ClientContext;
 using grpc::Status;
@@ -552,8 +550,9 @@ void mapPrototoResponse(const inference::ModelInferResponse& reply,
   }
 }
 
-InferenceResponse GrpcClient::modelInfer(const std::string& model,
-                                         const InferenceRequest& request) {
+InferenceResponse runInference(inference::GRPCInferenceService::Stub* stub,
+                               const std::string& model,
+                               const InferenceRequest& request) {
   inference::ModelInferRequest grpc_request;
   inference::ModelInferResponse reply;
 
@@ -562,7 +561,6 @@ InferenceResponse GrpcClient::modelInfer(const std::string& model,
   grpc_request.set_model_name(model);
   mapRequestToProto(request, grpc_request);
 
-  auto* stub = this->impl_->getStub();
   Status status = stub->ModelInfer(&context, grpc_request, &reply);
 
   if (!status.ok()) {
@@ -574,6 +572,16 @@ InferenceResponse GrpcClient::modelInfer(const std::string& model,
   return response;
 }
 
+InferenceResponseFuture GrpcClient::modelInferAsync(
+  const std::string& model, const InferenceRequest& request) {
+  return std::async(runInference, this->impl_->getStub(), model, request);
+}
+
+InferenceResponse GrpcClient::modelInfer(const std::string& model,
+                                         const InferenceRequest& request) {
+  return runInference(this->impl_->getStub(), model, request);
+}
+
 bool GrpcClient::hasHardware(const std::string& name, int num) {
   inference::HasHardwareRequest grpc_request;
   inference::HasHardwareResponse reply;
diff --git a/src/proteus/clients/http.cpp b/src/proteus/clients/http.cpp
index d481b20f3..e99f9ef7a 100644
--- a/src/proteus/clients/http.cpp
+++ b/src/proteus/clients/http.cpp
@@ -20,52 +20,56 @@
 #include "proteus/clients/http.hpp"
 
 #include <drogon/HttpClient.h>            // for HttpClient, HttpClientPtr
-#include <drogon/HttpRequest.h>           // for HttpRequest
+#include <drogon/HttpRequest.h>           // for HttpRequest, HttpReques...
 #include <drogon/HttpResponse.h>          // for HttpResponse
-#include <drogon/HttpTypes.h>             // for Get, ReqResult, k200OK
+#include <drogon/HttpTypes.h>             // for k200OK, Get, Post, ReqR...
 #include <json/value.h>                   // for Value, arrayValue, obje...
-#include <trantor/net/EventLoop.h>        // for EventLoop
 #include <trantor/net/EventLoopThread.h>  // for EventLoopThread
 
-#include <cassert>      // for assert
-#include <set>          // for set
-#include <stdexcept>    // for invalid_argument, runti...
-#include <string_view>  // for string_view
-#include <thread>       // for thread
-#include <utility>      // for tuple_element<>::type
+#include <cassert>        // for assert
+#include <unordered_set>  // for unordered_set
+#include <utility>        // for tuple_element<>::type
 
-#include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_HTTP
-#include "proteus/clients/http_internal.hpp"  // for mapJsonToResponse, mapP...
+#include "proteus/clients/http_internal.hpp"  // for mapParametersToJson
 #include "proteus/core/exceptions.hpp"        // for bad_status
 
 namespace proteus {
 
+void addHeaders(drogon::HttpRequestPtr req, const StringMap& headers) {
+  for (const auto& [field, value] : headers) {
+    req->addHeader(field, value);
+  }
+}
+
 class HttpClient::HttpClientImpl {
  public:
-  explicit HttpClientImpl(const std::string& address, const StringMap& headers)
-    : headers_(headers) {
+  explicit HttpClientImpl(const std::string& address) {
     loop_.run();
     client_ = drogon::HttpClient::newHttpClient(address, loop_.getLoop());
   }
 
   drogon::HttpClient* getClient() { return client_.get(); }
 
-  void addHeaders(drogon::HttpRequestPtr req) const {
-    for (const auto& [field, value] : headers_) {
-      req->addHeader(field, value);
-    }
-  }
-
  private:
   trantor::EventLoopThread loop_;
   drogon::HttpClientPtr client_;
-  StringMap headers_;
 };
 
-HttpClient::HttpClient(const std::string& address, const StringMap& headers) {
-  this->impl_ = std::make_unique<HttpClient::HttpClientImpl>(address, headers);
+HttpClient::HttpClient(std::string address, const StringMap& headers)
+  : address_(std::move(address)), headers_(headers) {
+  this->impl_ = std::make_unique<HttpClient::HttpClientImpl>(address_);
 }
 
+HttpClient::HttpClient(const HttpClient& other)
+  : address_(other.address_), headers_(other.headers_) {
+  this->impl_ = std::make_unique<HttpClient::HttpClientImpl>(address_);
+}
+
+HttpClient::HttpClient(HttpClient&& other) noexcept
+  : address_(std::move(other.address_)),
+    headers_(std::move(other.headers_)),
+    impl_(std::move(other.impl_)) {}
+
 // needed for HttpClientImpl forward declaration in WebSocket client
 HttpClient::~HttpClient() = default;
 
@@ -94,7 +98,7 @@ ServerMetadata HttpClient::serverMetadata() {
   auto req = drogon::HttpRequest::newHttpRequest();
   req->setMethod(drogon::Get);
   req->setPath("/v2");
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -119,7 +123,7 @@ bool HttpClient::serverLive() {
   req->setMethod(drogon::Get);
   auto path = "/v2/health/live";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   if (result != drogon::ReqResult::Ok) {
@@ -135,7 +139,7 @@ bool HttpClient::serverReady() {
   req->setMethod(drogon::Get);
   auto path = "/v2/health/ready";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -149,7 +153,7 @@ bool HttpClient::modelReady(const std::string& model) {
   req->setMethod(drogon::Get);
   auto path = "/v2/models/" + model + "/ready";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -162,7 +166,7 @@ ModelMetadata HttpClient::modelMetadata(const std::string& model) {
   req->setMethod(drogon::Get);
   auto path = "/v2/models/" + model;
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -183,7 +187,7 @@ void HttpClient::modelLoad(const std::string& model,
   req->setMethod(drogon::Post);
   auto path = "/v2/repository/models/" + model + "/load";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -200,7 +204,7 @@ void HttpClient::modelUnload(const std::string& model) {
   req->setMethod(drogon::Post);
   auto path = "/v2/repository/models/" + model + "/unload";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -223,7 +227,7 @@ std::string HttpClient::workerLoad(const std::string& model,
   req->setMethod(drogon::Post);
   auto path = "/v2/workers/" + model + "/load";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -241,7 +245,7 @@ void HttpClient::workerUnload(const std::string& model) {
   req->setMethod(drogon::Post);
   auto path = "/v2/workers/" + model + "/unload";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -251,10 +255,10 @@ void HttpClient::workerUnload(const std::string& model) {
   }
 }
 
-InferenceResponse HttpClient::modelInfer(const std::string& model,
-                                         const InferenceRequest& request) {
-  auto* client = this->impl_->getClient();
-
+InferenceResponse runInference(drogon::HttpClient* client,
+                               const std::string& model,
+                               const InferenceRequest& request,
+                               const StringMap& headers) {
   assert(!request.getInputs().empty());
 
   auto json = mapRequestToJson(request);
@@ -262,7 +266,7 @@ InferenceResponse HttpClient::modelInfer(const std::string& model,
   req->setMethod(drogon::Post);
   auto path = "/v2/models/" + model + "/infer";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -274,6 +278,17 @@ InferenceResponse HttpClient::modelInfer(const std::string& model,
   return mapJsonToResponse(resp.get());
 }
 
+InferenceResponseFuture HttpClient::modelInferAsync(
+  const std::string& model, const InferenceRequest& request) {
+  return std::async(runInference, this->impl_->getClient(), model, request,
+                    headers_);
+}
+
+InferenceResponse HttpClient::modelInfer(const std::string& model,
+                                         const InferenceRequest& request) {
+  return runInference(this->impl_->getClient(), model, request, headers_);
+}
+
 std::vector<std::string> HttpClient::modelList() {
   auto* client = this->impl_->getClient();
 
@@ -281,7 +296,7 @@ std::vector<std::string> HttpClient::modelList() {
   req->setMethod(drogon::Get);
   const std::string path = "/v2/models";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
@@ -309,11 +324,17 @@ bool HttpClient::hasHardware(const std::string& name, int num) {
   req->setMethod(drogon::Get);
   const std::string path = "/v2/hardware";
   req->setPath(path);
-  impl_->addHeaders(req);
+  addHeaders(req, headers_);
 
   auto [result, response] = client->sendRequest(req);
   check_error(result);
   return response->statusCode() == drogon::k200OK;
 }
 
+const std::string& HttpClient::getAddress() const& { return address_; }
+std::string HttpClient::getAddress() const&& { return address_; }
+
+const StringMap& HttpClient::getHeaders() const& { return headers_; }
+StringMap HttpClient::getHeaders() const&& { return headers_; }
+
 }  // namespace proteus
diff --git a/src/proteus/clients/http_internal.cpp b/src/proteus/clients/http_internal.cpp
index 24050b553..17fbbfd3d 100644
--- a/src/proteus/clients/http_internal.cpp
+++ b/src/proteus/clients/http_internal.cpp
@@ -422,12 +422,9 @@ using OutputBuilder =
   InferenceRequestOutputBuilder<std::shared_ptr<Json::Value>>;
 
 InferenceRequestPtr RequestBuilder::build(
-  const std::shared_ptr<Json::Value> &req, size_t &buffer_index,
-  const std::vector<BufferRawPtrs> &input_buffers,
-  std::vector<size_t> &input_offsets,
-  const std::vector<BufferRawPtrs> &output_buffers,
-  std::vector<size_t> &output_offsets, const size_t &batch_size,
-  size_t &batch_offset) {
+  const std::shared_ptr<Json::Value> &req, const BufferRawPtrs &input_buffers,
+  std::vector<size_t> &input_offsets, const BufferRawPtrs &output_buffers,
+  std::vector<size_t> &output_offsets) {
   auto request = std::make_shared<InferenceRequest>();
 
   if (req->isMember("id")) {
@@ -452,63 +449,53 @@ InferenceRequestPtr RequestBuilder::build(
 
   request->callback_ = nullptr;
 
-  auto buffer_index_backup = buffer_index;
-  auto batch_offset_backup = batch_offset;
   for (auto const &i : inputs) {
     if (!i.isObject()) {
       throw invalid_argument("At least one element in 'inputs' is not an obj");
     }
-    const auto &buffers = input_buffers[buffer_index];
+    const auto &buffers = input_buffers;
+    auto index = 0;
     for (const auto &buffer : buffers) {
-      auto &offset = input_offsets[buffer_index];
+      auto &offset = input_offsets[index];
 
       auto input =
         InputBuilder::build(std::make_shared<Json::Value>(i), buffer, offset);
       offset += (input.getSize() * input.getDatatype().size());
 
       request->inputs_.push_back(std::move(input));
-    }
-    batch_offset++;
-    if (batch_offset == batch_size) {
-      batch_offset = 0;
-      buffer_index++;
-      // std::fill(input_offsets.begin(), input_offsets.end(), 0);
+      index++;
     }
   }
 
   // TODO(varunsh): output_offset is currently ignored! The size of the output
   // needs to come from the worker but we have no such information.
-  buffer_index = buffer_index_backup;
-  batch_offset = batch_offset_backup;
   if (req->isMember("outputs")) {
     auto outputs = req->get("outputs", Json::arrayValue);
     for (auto const &i : outputs) {
-      auto buffers = output_buffers[buffer_index];
+      const auto &buffers = output_buffers;
+      auto index = 0;
       for (auto &buffer : buffers) {
-        auto &offset = output_offsets[buffer_index];
+        auto &offset = output_offsets[index];
 
         auto output = OutputBuilder::build(std::make_shared<Json::Value>(i));
         output.setData(static_cast<std::byte *>(buffer->data()) + offset);
         request->outputs_.push_back(std::move(output));
         // output += request->outputs_.back().getSize(); // see TODO
+        index++;
       }
     }
   } else {
     for (auto const &i : inputs) {
       (void)i;  // suppress unused variable warning
-      const auto &buffers = output_buffers[buffer_index];
+      const auto &buffers = output_buffers;
+      auto index = 0;
       for (const auto &buffer : buffers) {
-        const auto &offset = output_offsets[buffer_index];
+        const auto &offset = output_offsets[index];
 
         request->outputs_.emplace_back();
         request->outputs_.back().setData(
           static_cast<std::byte *>(buffer->data()) + offset);
-      }
-      batch_offset++;
-      if (batch_offset == batch_size) {
-        batch_offset = 0;
-        buffer_index++;
-        std::fill(output_offsets.begin(), output_offsets.end(), 0);
+        index++;
       }
     }
   }
@@ -715,18 +702,15 @@ void propagate(drogon::HttpResponse *resp, const StringMap &context) {
 #endif
 
 std::shared_ptr<InferenceRequest> DrogonHttp::getRequest(
-  size_t &buffer_index, const std::vector<BufferRawPtrs> &input_buffers,
-  std::vector<size_t> &input_offsets,
-  const std::vector<BufferRawPtrs> &output_buffers,
-  std::vector<size_t> &output_offsets, const size_t &batch_size,
-  size_t &batch_offset) {
+  const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+  const BufferRawPtrs &output_buffers, std::vector<size_t> &output_offsets) {
 #ifdef PROTEUS_ENABLE_LOGGING
   const auto &logger = this->getLogger();
 #endif
   try {
-    auto request = RequestBuilder::build(
-      this->json_, buffer_index, input_buffers, input_offsets, output_buffers,
-      output_offsets, batch_size, batch_offset);
+    auto request =
+      RequestBuilder::build(this->json_, input_buffers, input_offsets,
+                            output_buffers, output_offsets);
     Callback callback = [callback = std::move(this->callback_)](
                           const InferenceResponse &response) {
       drogon::HttpResponsePtr resp;
diff --git a/src/proteus/clients/http_internal.hpp b/src/proteus/clients/http_internal.hpp
index a1550db01..2f5d25a08 100644
--- a/src/proteus/clients/http_internal.hpp
+++ b/src/proteus/clients/http_internal.hpp
@@ -20,14 +20,14 @@
 #ifndef GUARD_PROTEUS_CLIENTS_HTTP_INTERNAL
 #define GUARD_PROTEUS_CLIENTS_HTTP_INTERNAL
 
-#include <drogon/HttpRequest.h>           // for HttpRequestPtr
-#include <drogon/utils/FunctionTraits.h>  // for HttpResponsePtr
-#include <json/value.h>                   // for Value
+#include <drogon/HttpRequest.h>   // for HttpRequestPtr
+#include <drogon/HttpResponse.h>  // for HttpResponsePtr
+#include <json/value.h>           // for Value
 
 #include <cstddef>     // for size_t
+#include <exception>   // for invalid_argument
 #include <functional>  // for function
 #include <memory>      // for shared_ptr
-#include <stdexcept>   // for invalid_argument
 #include <string>      // for string
 #include <vector>      // for vector
 
@@ -36,10 +36,6 @@
 #include "proteus/core/predict_api_internal.hpp"  // for InferenceRequestBui...
 #include "proteus/helpers/declarations.hpp"       // for BufferRawPtrs, Infe...
 
-namespace drogon {
-class HttpResponse;
-}
-
 namespace proteus {
 
 /**
@@ -63,13 +59,11 @@ Json::Value mapRequestToJson(const InferenceRequest &request);
 template <>
 class InferenceRequestBuilder<std::shared_ptr<Json::Value>> {
  public:
-  static InferenceRequestPtr build(
-    const std::shared_ptr<Json::Value> &req, size_t &buffer_index,
-    const std::vector<BufferRawPtrs> &input_buffers,
-    std::vector<size_t> &input_offsets,
-    const std::vector<BufferRawPtrs> &output_buffers,
-    std::vector<size_t> &output_offsets, const size_t &batch_size,
-    size_t &batch_offset);
+  static InferenceRequestPtr build(const std::shared_ptr<Json::Value> &req,
+                                   const BufferRawPtrs &input_buffers,
+                                   std::vector<size_t> &input_offsets,
+                                   const BufferRawPtrs &output_buffers,
+                                   std::vector<size_t> &output_offsets);
 };
 
 using RequestBuilder = InferenceRequestBuilder<std::shared_ptr<Json::Value>>;
@@ -96,11 +90,9 @@ class DrogonHttp : public Interface {
   DrogonHttp(const drogon::HttpRequestPtr &req, DrogonCallback callback);
 
   std::shared_ptr<InferenceRequest> getRequest(
-    size_t &buffer_index, const std::vector<BufferRawPtrs> &input_buffers,
-    std::vector<size_t> &input_offsets,
-    const std::vector<BufferRawPtrs> &output_buffers,
-    std::vector<size_t> &output_offsets, const size_t &batch_size,
-    size_t &batch_offset) override;
+    const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+    const BufferRawPtrs &output_buffers,
+    std::vector<size_t> &output_offsets) override;
 
   size_t getInputSize() override;
   void errorHandler(const std::exception &e) override;
diff --git a/src/proteus/clients/native.cpp b/src/proteus/clients/native.cpp
index 47d491968..d394a8c88 100644
--- a/src/proteus/clients/native.cpp
+++ b/src/proteus/clients/native.cpp
@@ -20,28 +20,19 @@
 
 #include "proteus/clients/native.hpp"
 
-#include <cstdlib>        // for getenv
-#include <future>         // for promise
-#include <memory>         // for unique_ptr, make_unique
-#include <stdexcept>      // for invalid_argument
-#include <string>         // for string, basic_string
-#include <unordered_map>  // for unordered_map, operat...
-#include <unordered_set>  // for unordered_set
-#include <utility>        // for move, pair, make_pair
-
-#include "proteus/batching/batcher.hpp"         // for Batcher
+#include <future>   // for future, promise
+#include <memory>   // for unique_ptr, make_unique
+#include <string>   // for string
+#include <utility>  // for move
+
 #include "proteus/build_options.hpp"            // for PROTEUS_ENABLE_TRACING
 #include "proteus/clients/native_internal.hpp"  // for CppNativeApi
-#include "proteus/core/api.hpp"                 // for modelLoad
-#include "proteus/core/exceptions.hpp"          // for bad_status
-#include "proteus/core/manager.hpp"             // for Manager
-#include "proteus/core/worker_info.hpp"         // for WorkerInfo
-#include "proteus/helpers/exec.hpp"             // for exec
+#include "proteus/core/api.hpp"                 // for modelLoad, workerLoad
+#include "proteus/core/exceptions.hpp"          // for invalid_argument
+#include "proteus/core/interface.hpp"           // for Interface
 #include "proteus/helpers/string.hpp"           // for toLower
-#include "proteus/observation/logging.hpp"      // for initLogging
 #include "proteus/observation/metrics.hpp"      // for Metrics, MetricCounte...
-#include "proteus/observation/tracing.hpp"      // for startTrace, startTracer
-#include "proteus/version.hpp"                  // for kProteusVersion
+#include "proteus/observation/tracing.hpp"      // for startTrace, Trace
 
 namespace proteus {
 
@@ -76,8 +67,8 @@ std::string NativeClient::workerLoad(const std::string& model,
   return ::proteus::workerLoad(model_lower, parameters);
 }
 
-InferenceResponseFuture NativeClient::enqueue(const std::string& workerName,
-                                              InferenceRequest request) {
+InferenceResponseFuture NativeClient::modelInferAsync(
+  const std::string& workerName, const InferenceRequest& request) {
 #ifdef PROTEUS_ENABLE_METRICS
   Metrics::getInstance().incrementCounter(MetricCounterIDs::kCppNative);
 #endif
@@ -86,7 +77,7 @@ InferenceResponseFuture NativeClient::enqueue(const std::string& workerName,
   auto trace = startTrace(&(__func__[0]));
   trace->startSpan("C++ enqueue");
 #endif
-  auto api = std::make_unique<CppNativeApi>(std::move(request));
+  auto api = std::make_unique<CppNativeApi>(request);
   auto future = api->getPromise()->get_future();
 #ifdef PROTEUS_ENABLE_TRACING
   trace->endSpan();
@@ -99,7 +90,7 @@ InferenceResponseFuture NativeClient::enqueue(const std::string& workerName,
 
 InferenceResponse NativeClient::modelInfer(const std::string& model,
                                            const InferenceRequest& request) {
-  auto future = enqueue(model, request);
+  auto future = modelInferAsync(model, request);
   return future.get();
 }
 
diff --git a/src/proteus/clients/native_internal.cpp b/src/proteus/clients/native_internal.cpp
index 603b81a3b..19830e893 100644
--- a/src/proteus/clients/native_internal.cpp
+++ b/src/proteus/clients/native_internal.cpp
@@ -19,18 +19,17 @@
 
 #include "proteus/clients/native_internal.hpp"
 
-#include <algorithm>   // for fill
-#include <cstddef>     // for size_t, byte
+#include <cstddef>     // for byte, size_t
 #include <cstdint>     // for uint64_t
 #include <cstring>     // for memcpy
-#include <functional>  // for _Bind_helper<>::type, _Placeh...
+#include <functional>  // for multiplies
 #include <numeric>     // for accumulate
 #include <string>      // for string
 #include <utility>     // for move
 
 #include "proteus/buffers/buffer.hpp"       // for Buffer
-#include "proteus/core/data_types.hpp"      // for getSize
-#include "proteus/observation/logging.hpp"  // for Logger
+#include "proteus/core/data_types.hpp"      // for DataType
+#include "proteus/observation/logging.hpp"  // for Logger, PROTEUS_LOG_ERROR
 
 namespace proteus {
 template <typename T>
@@ -71,76 +70,57 @@ using InputBuilder = InferenceRequestInputBuilder<InferenceRequestInput>;
 template <>
 class InferenceRequestBuilder<InferenceRequest> {
  public:
-  static InferenceRequestPtr build(
-    const InferenceRequest &req, size_t &buffer_index,
-    const std::vector<BufferRawPtrs> &input_buffers,
-    std::vector<size_t> &input_offsets,
-    const std::vector<BufferRawPtrs> &output_buffers,
-    std::vector<size_t> &output_offsets, const size_t &batch_size,
-    size_t &batch_offset) {
+  static InferenceRequestPtr build(const InferenceRequest &req,
+                                   const BufferRawPtrs &input_buffers,
+                                   std::vector<size_t> &input_offsets,
+                                   const BufferRawPtrs &output_buffers,
+                                   std::vector<size_t> &output_offsets) {
     auto request = std::make_shared<InferenceRequest>();
 
     request->id_ = req.id_;
     request->parameters_ = req.parameters_;
     request->callback_ = nullptr;
 
-    auto buffer_index_backup = buffer_index;
-    auto batch_offset_backup = batch_offset;
-
     for (const auto &input : req.inputs_) {
-      const auto &buffers = input_buffers[buffer_index];
+      const auto &buffers = input_buffers;
+      auto index = 0;
       for (auto &buffer : buffers) {
-        auto &offset = input_offsets[buffer_index];
+        auto &offset = input_offsets[index];
 
         request->inputs_.push_back(InputBuilder::build(input, buffer, offset));
         const auto &last_input = request->inputs_.back();
         offset += (last_input.getSize() * last_input.getDatatype().size());
-      }
-      batch_offset++;
-      if (batch_offset == batch_size) {
-        batch_offset = 0;
-        buffer_index++;
-        // std::fill(input_offsets.begin(), input_offsets.end(), 0);
+        index++;
       }
     }
 
     // TODO(varunsh): output_offset is currently ignored! The size of the output
     // needs to come from the worker but we have no such information.
-    buffer_index = buffer_index_backup;
-    batch_offset = batch_offset_backup;
     if (!req.outputs_.empty()) {
       for (const auto &output : req.outputs_) {
-        const auto &buffers = output_buffers[buffer_index];
+        const auto &buffers = output_buffers;
+        auto index = 0;
         for (auto &buffer : buffers) {
-          const auto &offset = output_offsets[buffer_index];
+          const auto &offset = output_offsets[index];
 
           request->outputs_.emplace_back(output);
           request->outputs_.back().setData(
             static_cast<std::byte *>(buffer->data()) + offset);
-        }
-        batch_offset++;
-        if (batch_offset == batch_size) {
-          batch_offset = 0;
-          buffer_index++;
-          std::fill(output_offsets.begin(), output_offsets.end(), 0);
+          index++;
         }
       }
     } else {
       for (const auto &input : req.inputs_) {
         (void)input;  // suppress unused variable warning
-        const auto &buffers = output_buffers[buffer_index];
+        const auto &buffers = output_buffers;
+        auto index = 0;
         for (auto &buffer : buffers) {
-          const auto &offset = output_offsets[buffer_index];
+          const auto &offset = output_offsets[index];
 
           request->outputs_.emplace_back();
           request->outputs_.back().setData(
             static_cast<std::byte *>(buffer->data()) + offset);
-        }
-        batch_offset++;
-        if (batch_offset == batch_size) {
-          batch_offset = 0;
-          buffer_index++;
-          std::fill(output_offsets.begin(), output_offsets.end(), 0);
+          index++;
         }
       }
     }
@@ -168,14 +148,11 @@ std::promise<proteus::InferenceResponse> *CppNativeApi::getPromise() {
 // }
 
 std::shared_ptr<InferenceRequest> CppNativeApi::getRequest(
-  size_t &buffer_index, const std::vector<BufferRawPtrs> &input_buffers,
-  std::vector<size_t> &input_offsets,
-  const std::vector<BufferRawPtrs> &output_buffers,
-  std::vector<size_t> &output_offsets, const size_t &batch_size,
-  size_t &batch_offset) {
-  auto request = RequestBuilder::build(
-    this->request_, buffer_index, input_buffers, input_offsets, output_buffers,
-    output_offsets, batch_size, batch_offset);
+  const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+  const BufferRawPtrs &output_buffers, std::vector<size_t> &output_offsets) {
+  auto request =
+    RequestBuilder::build(this->request_, input_buffers, input_offsets,
+                          output_buffers, output_offsets);
   Callback callback =
     [promise = std::move(this->promise_)](const InferenceResponse &response) {
       promise->set_value(response);
diff --git a/src/proteus/clients/native_internal.hpp b/src/proteus/clients/native_internal.hpp
index 0c3720eb6..377077a6d 100644
--- a/src/proteus/clients/native_internal.hpp
+++ b/src/proteus/clients/native_internal.hpp
@@ -21,9 +21,9 @@
 #define GUARD_PROTEUS_CLIENTS_NATIVE_INTERNAL
 
 #include <cstddef>    // for size_t
+#include <exception>  // for exception
 #include <future>     // for promise
 #include <memory>     // for shared_ptr
-#include <stdexcept>  // for invalid_argument
 #include <vector>     // for vector
 
 #include "proteus/core/interface.hpp"        // for Interface
@@ -37,11 +37,9 @@ class CppNativeApi : public Interface {
   explicit CppNativeApi(InferenceRequest request);
 
   std::shared_ptr<InferenceRequest> getRequest(
-    size_t &buffer_index, const std::vector<BufferRawPtrs> &input_buffers,
-    std::vector<size_t> &input_offsets,
-    const std::vector<BufferRawPtrs> &output_buffers,
-    std::vector<size_t> &output_offsets, const size_t &batch_size,
-    size_t &batch_offset) override;
+    const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+    const BufferRawPtrs &output_buffers,
+    std::vector<size_t> &output_offsets) override;
 
   size_t getInputSize() override;
   void errorHandler(const std::exception &e) override;
diff --git a/src/proteus/clients/websocket.cpp b/src/proteus/clients/websocket.cpp
index 9f4eb82d7..43ff55d10 100644
--- a/src/proteus/clients/websocket.cpp
+++ b/src/proteus/clients/websocket.cpp
@@ -206,6 +206,12 @@ void WebSocketClient::workerUnload(const std::string& model) {
   client->workerUnload(model);
 }
 
+InferenceResponseFuture WebSocketClient::modelInferAsync(
+  const std::string& model, const InferenceRequest& request) {
+  auto* client = this->impl_->getHttpClient();
+  return client->modelInferAsync(model, request);
+}
+
 InferenceResponse WebSocketClient::modelInfer(const std::string& model,
                                               const InferenceRequest& request) {
   auto* client = this->impl_->getHttpClient();
@@ -222,8 +228,8 @@ bool WebSocketClient::hasHardware(const std::string& name, int num) {
   return client->hasHardware(name, num);
 }
 
-void WebSocketClient::modelInferAsync(const std::string& model,
-                                      const InferenceRequest& request) {
+void WebSocketClient::modelInferWs(const std::string& model,
+                                   const InferenceRequest& request) {
   auto* client = this->impl_->getWsClient();
 
   auto json = mapRequestToJson(request);
diff --git a/src/proteus/core/api.cpp b/src/proteus/core/api.cpp
index 74878bfac..4fdce9e7e 100644
--- a/src/proteus/core/api.cpp
+++ b/src/proteus/core/api.cpp
@@ -19,23 +19,30 @@
 
 #include "proteus/core/api.hpp"
 
-#include <json/reader.h>  // for CharReaderBuilder
-#include <json/value.h>
-
-#include <algorithm>
-#include <string>
-
-#include "proteus/batching/batcher.hpp"
-#include "proteus/build_options.hpp"
-#include "proteus/core/interface.hpp"
-#include "proteus/core/manager.hpp"
-#include "proteus/core/model_repository.hpp"
-#include "proteus/core/predict_api.hpp"
-#include "proteus/core/worker_info.hpp"
-#include "proteus/version.hpp"  // for kProteusVersion
+#include <json/reader.h>  // for CharReaderBuilder, Char...
+#include <json/value.h>   // for Value
+
+#include <cassert>        // for assert
+#include <cctype>         // for tolower
+#include <iostream>       // for operator<<, basic_ostream
+#include <string>         // for string, operator+, stoi
+#include <unordered_map>  // for operator==, unordered_m...
+#include <unordered_set>  // for unordered_set
+#include <utility>        // for move, pair
+
+#include "proteus/batching/batcher.hpp"       // for Batcher
+#include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_VITIS
+#include "proteus/core/exceptions.hpp"        // for external_error, invalid...
+#include "proteus/core/interface.hpp"         // IWYU pragma: keep
+#include "proteus/core/manager.hpp"           // for Manager
+#include "proteus/core/model_repository.hpp"  // for ModelRepository
+#include "proteus/core/predict_api.hpp"       // for ServerMetadata, ModelMe...
+#include "proteus/core/worker_info.hpp"       // for WorkerInfo
+#include "proteus/version.hpp"                // for kProteusVersion
 
 #ifdef PROTEUS_ENABLE_VITIS
-#include <sockpp/tcp_connector.h>
+#include <sockpp/socket.h>         // for socket, socket_initializer
+#include <sockpp/tcp_connector.h>  // for tcp_connector
 #endif
 
 namespace proteus {
diff --git a/src/proteus/core/api.hpp b/src/proteus/core/api.hpp
index bcd5dcf13..ba172b903 100644
--- a/src/proteus/core/api.hpp
+++ b/src/proteus/core/api.hpp
@@ -30,7 +30,7 @@ namespace proteus {
 
 class ModelMetadata;
 class RequestParameters;
-class ServerMetadata;
+struct ServerMetadata;
 class Interface;
 
 void modelLoad(const std::string& model, RequestParameters* parameters);
diff --git a/src/proteus/core/interface.cpp b/src/proteus/core/interface.cpp
index a0032ae92..f082ff588 100644
--- a/src/proteus/core/interface.cpp
+++ b/src/proteus/core/interface.cpp
@@ -21,9 +21,8 @@
 
 #include <utility>  // for move
 
-#include "proteus/build_options.hpp"        // for PROTEUS_ENABLE_TRACING
-#include "proteus/observation/logging.hpp"  // for getLogger, LoggerPtr
-#include "proteus/observation/tracing.hpp"  // for SpanPtr
+#include "proteus/build_options.hpp"        // for PROTEUS_ENABLE_LOGGING
+#include "proteus/observation/tracing.hpp"  // for TracePtr
 
 namespace proteus {
 
diff --git a/src/proteus/core/interface.hpp b/src/proteus/core/interface.hpp
index 9297893af..f77b31124 100644
--- a/src/proteus/core/interface.hpp
+++ b/src/proteus/core/interface.hpp
@@ -22,14 +22,14 @@
 
 #include <chrono>     // for high_resolution_clock
 #include <cstddef>    // for size_t
-#include <memory>     // for shared_ptr, unique_ptr
-#include <stdexcept>  // for invalid_argument
+#include <exception>  // for exception
+#include <memory>     // for shared_ptr
 #include <vector>     // for vector
 
-#include "proteus/build_options.hpp"         // for PROTEUS_ENABLE_TRACING
+#include "proteus/build_options.hpp"         // for PROTEUS_ENABLE_LOGGING
 #include "proteus/helpers/declarations.hpp"  // for BufferRawPtrs
-#include "proteus/observation/logging.hpp"   // for LoggerPtr
-#include "proteus/observation/tracing.hpp"   // for SpanPtr, Span
+#include "proteus/observation/logging.hpp"   // for Logger, Loggers, Loggers...
+#include "proteus/observation/tracing.hpp"   // for TracePtr
 
 namespace proteus {
 class InferenceRequest;
@@ -80,21 +80,16 @@ class Interface {
   /**
    * @brief Construct an InferenceRequest using the data in the Interface
    *
-   * @param buffer_index the index to use for the vector of I/O buffers
    * @param input_buffers a vector of buffers to hold the input data
    * @param input_offsets offsets of where to start storing input data
    * @param output_buffers a vector of buffers to hold the output data
    * @param output_offsets offsets of where to start storing output data
-   * @param batch_size the requested batch size
-   * @param batch_offset the current batch size
    * @return std::shared_ptr<InferenceRequest>
    */
   virtual std::shared_ptr<InferenceRequest> getRequest(
-    size_t &buffer_index, const std::vector<BufferRawPtrs> &input_buffers,
-    std::vector<size_t> &input_offsets,
-    const std::vector<BufferRawPtrs> &output_buffers,
-    std::vector<size_t> &output_offsets, const size_t &batch_size,
-    size_t &batch_offset) = 0;
+    const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+    const BufferRawPtrs &output_buffers,
+    std::vector<size_t> &output_offsets) = 0;
 
   /**
    * @brief Given an exception, handle the exception appropriately.
@@ -114,6 +109,7 @@ class Interface {
 #ifdef PROTEUS_ENABLE_LOGGING
   const Logger &getLogger() const;
 #endif
+
  private:
 #ifdef PROTEUS_ENABLE_LOGGING
   Logger logger_{Loggers::kServer};
diff --git a/src/proteus/core/manager.cpp b/src/proteus/core/manager.cpp
index 5b7e4b0c3..3d3c5b503 100644
--- a/src/proteus/core/manager.cpp
+++ b/src/proteus/core/manager.cpp
@@ -20,11 +20,11 @@
 
 #include "proteus/core/manager.hpp"
 
-#include <stdexcept>  // for invalid_argument
-#include <thread>     // for yield, thread
-#include <utility>    // for pair, make_pair, move
+#include <thread>   // for yield, thread
+#include <utility>  // for pair, make_pair, move
 
-#include "proteus/build_options.hpp"     // for PROTEUS_ENABLE_LOGGING, kMax...
+#include "proteus/build_options.hpp"     // for kMaxModelNameSize
+#include "proteus/core/exceptions.hpp"   // for invalid_argument
 #include "proteus/core/worker_info.hpp"  // for WorkerInfo
 #include "proteus/helpers/thread.hpp"    // for setThreadName
 #include "proteus/workers/worker.hpp"    // for Worker
diff --git a/src/proteus/core/manager.hpp b/src/proteus/core/manager.hpp
index c219853b3..8c2c62abd 100644
--- a/src/proteus/core/manager.hpp
+++ b/src/proteus/core/manager.hpp
@@ -32,13 +32,12 @@
 
 #include "proteus/build_options.hpp"        // for PROTEUS_ENABLE_LOGGING
 #include "proteus/core/predict_api.hpp"     // for RequestParameters
+#include "proteus/core/worker_info.hpp"     // for WorkerInfo
 #include "proteus/helpers/queue.hpp"        // for BlockingConcurrentQueue
 #include "proteus/observation/logging.hpp"  // for LoggerPtr
 
-namespace proteus {
-class WorkerInfo;
-}
 // IWYU pragma: no_forward_declare proteus::RequestParameters
+// IWYU pragma: no_forward_declare proteus::WorkerInfo
 
 namespace proteus {
 
diff --git a/src/proteus/core/model_repository.cpp b/src/proteus/core/model_repository.cpp
index 33c92347a..2278d0e2f 100644
--- a/src/proteus/core/model_repository.cpp
+++ b/src/proteus/core/model_repository.cpp
@@ -14,20 +14,20 @@
 
 #include "proteus/core/model_repository.hpp"
 
-#include <fcntl.h>
-#include <google/protobuf/io/zero_copy_stream_impl.h>
-#include <google/protobuf/text_format.h>
-
-#include <cassert>
-#include <filesystem>
-#include <fstream>
-
-#include "model_config.pb.h"
-#include "proteus/core/exceptions.hpp"
-#include "proteus/core/manager.hpp"
-#include "proteus/core/predict_api.hpp"
-#include "proteus/core/worker_info.hpp"
-#include "proteus/observation/logging.hpp"
+#include <fcntl.h>                                     // for open, O_RDONLY
+#include <google/protobuf/io/zero_copy_stream_impl.h>  // for FileInputStream
+#include <google/protobuf/repeated_ptr_field.h>        // for RepeatedPtrField
+#include <google/protobuf/text_format.h>               // for TextFormat
+
+#include <chrono>      // for milliseconds
+#include <filesystem>  // for path, operator/
+#include <thread>      // for sleep_for
+
+#include "model_config.pb.h"                // for Config, InferP...
+#include "proteus/core/exceptions.hpp"      // for file_not_found...
+#include "proteus/core/manager.hpp"         // for Manager
+#include "proteus/core/predict_api.hpp"     // for RequestParameters
+#include "proteus/observation/logging.hpp"  // for Logger, PROTEU...
 
 namespace fs = std::filesystem;
 
diff --git a/src/proteus/core/worker_info.cpp b/src/proteus/core/worker_info.cpp
index 3f5d805f3..3f9ea3eb7 100644
--- a/src/proteus/core/worker_info.cpp
+++ b/src/proteus/core/worker_info.cpp
@@ -22,17 +22,14 @@
 
 #include <dlfcn.h>  // for dlerror, dlopen, dlsym, RTLD...
 
-#include <cctype>       // for toupper
-#include <climits>      // for UINT_MAX
-#include <cstdint>      // for int32_t
-#include <stdexcept>    // for invalid_argument
-#include <string>       // for string, operator+, basic_string
-#include <type_traits>  // for remove_reference<>::type
-#include <utility>      // for pair, move, make_pair
+#include <cctype>   // for toupper
+#include <climits>  // for UINT_MAX
+#include <cstdint>  // for int32_t
+#include <string>   // for string, operator+, basic_string
+#include <utility>  // for pair, move, make_pair
 
 #include "proteus/batching/batcher.hpp"  // for Batcher, BatcherStatus, Batc...
-#include "proteus/build_options.hpp"     // for PROTEUS_ENABLE_LOGGING
-#include "proteus/core/exceptions.hpp"   // for file_not_found_error
+#include "proteus/core/exceptions.hpp"   // for invalid_argument, file_not_f...
 #include "proteus/core/interface.hpp"    // IWYU pragma: keep
 #include "proteus/core/predict_api.hpp"  // for RequestParameters
 #include "proteus/workers/worker.hpp"    // for Worker, WorkerStatus, Worker...
diff --git a/src/proteus/main.cpp b/src/proteus/main.cpp
index 0aea68f99..f65f5917d 100644
--- a/src/proteus/main.cpp
+++ b/src/proteus/main.cpp
@@ -17,20 +17,21 @@
  * @brief Parses the command-line arguments and implements Proteus's entrypoint
  */
 
-#include <csignal>              // for signal, SIGINT
+#include <csignal>              // for signal, SIGINT, SIGTERM
 #include <cstdlib>              // for exit
-#include <cxxopts/cxxopts.hpp>  // for Options, value, OptionAdder
+#include <cxxopts/cxxopts.hpp>  // for value, OptionAdder, Opt...
 #include <efsw/efsw.hpp>        // for FileWatcher
-#include <filesystem>
-#include <iostream>  // for operator<<, endl, basic_o...
-#include <string>    // for string, allocator, operat...
-
-#include "proteus/build_options.hpp"   // for PROTEUS_ENABLE_HTTP, PROT...
-#include "proteus/clients/native.hpp"  // for initialize, terminate
-#include "proteus/core/model_repository.hpp"  // for ModelRepository
-#include "proteus/observation/logging.hpp"    // for logger
-#include "proteus/servers/grpc_server.hpp"    // for start, stop
-#include "proteus/servers/http_server.hpp"    // for start, stop
+#include <filesystem>           // for path, directory_iterator
+#include <iostream>             // for operator<<, basic_ostre...
+#include <memory>               // for unique_ptr, make_unique
+#include <string>               // for string, allocator, char...
+
+#include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_HTTP
+#include "proteus/clients/native.hpp"         // for NativeClient
+#include "proteus/core/exceptions.hpp"        // for runtime_error
+#include "proteus/core/model_repository.hpp"  // for UpdateListener, ModelRe...
+#include "proteus/observation/logging.hpp"    // for Logger, PROTEUS_LOG_INFO
+#include "proteus/servers/http_server.hpp"    // for stop
 #include "proteus/servers/server.hpp"         // for Server
 
 namespace fs = std::filesystem;
diff --git a/src/proteus/servers/grpc_server.cpp b/src/proteus/servers/grpc_server.cpp
index 306af2d48..418de0595 100644
--- a/src/proteus/servers/grpc_server.cpp
+++ b/src/proteus/servers/grpc_server.cpp
@@ -21,37 +21,34 @@
 
 #include <google/protobuf/repeated_ptr_field.h>  // for RepeatedPtrField
 #include <grpc/support/log.h>                    // for GPR_ASSERT, GPR_UNL...
-#include <grpcpp/grpcpp.h>                       // for Status, InsecureSer...
-
-#include <algorithm>   // for fill, max, transform
-#include <cstddef>     // for size_t, byte
-#include <cstdint>     // for uint64_t, int16_t
-#include <cstring>     // for memcpy
-#include <exception>   // for exception
-#include <functional>  // for _Bind_helper<>::type
-#include <iostream>    // for operator<<, cout
-#include <memory>      // for unique_ptr, shared_ptr
-#include <stdexcept>   // for invalid_argument
-#include <thread>      // for thread, yield
-#include <utility>     // for move
-#include <vector>      // for vector
+#include <grpcpp/grpcpp.h>                       // for ServerCompletionQueue
+
+#include <cstddef>        // for size_t, byte
+#include <cstdint>        // for uint64_t, int16_t
+#include <cstring>        // for memcpy
+#include <exception>      // for exception
+#include <functional>     // for _Bind_helper<>::type
+#include <iostream>       // for operator<<, cout
+#include <memory>         // for unique_ptr, shared_ptr
+#include <string>         // for allocator, string
+#include <thread>         // for thread, yield
+#include <unordered_set>  // for unordered_set
+#include <utility>        // for move
+#include <vector>         // for vector
 
 #include "predict_api.grpc.pb.h"                  // for GRPCInferenceServic...
 #include "predict_api.pb.h"                       // for InferTensorContents
-#include "proteus/batching/batcher.hpp"           // for Batcher
 #include "proteus/buffers/buffer.hpp"             // for Buffer
-#include "proteus/build_options.hpp"              // for PROTEUS_ENABLE_TRACING
+#include "proteus/build_options.hpp"              // for PROTEUS_ENABLE_LOGGING
 #include "proteus/clients/grpc_internal.hpp"      // for mapProtoToParameters
-#include "proteus/clients/native.hpp"             // for NativeClient
-#include "proteus/core/api.hpp"                   // for modelLoad
-#include "proteus/core/data_types.hpp"            // for DataType, mapStrToType
+#include "proteus/core/api.hpp"                   // for hasHardware, modelI...
+#include "proteus/core/data_types.hpp"            // for DataType, DataType:...
+#include "proteus/core/exceptions.hpp"            // for invalid_argument
 #include "proteus/core/interface.hpp"             // for Interface, Interfac...
-#include "proteus/core/model_repository.hpp"      // for ModelRepository
 #include "proteus/core/predict_api_internal.hpp"  // for InferenceRequestInput
-#include "proteus/core/worker_info.hpp"           // for WorkerInfo
 #include "proteus/helpers/declarations.hpp"       // for BufferRawPtrs, Infe...
 #include "proteus/helpers/string.hpp"             // for toLower
-#include "proteus/observation/logging.hpp"        // for Logger
+#include "proteus/observation/logging.hpp"        // for Logger, Loggers
 #include "proteus/observation/tracing.hpp"        // for Trace, startTrace
 
 namespace proteus {
@@ -65,6 +62,8 @@ class CallDataWorkerUnload;
 class CallDataServerLive;
 class CallDataServerMetadata;
 class CallDataServerReady;
+class CallDataHasHardware;
+class CallDataModelList;
 }  // namespace proteus
 
 // use aliases to prevent clashes between grpc:: and proteus::grpc::
@@ -348,7 +347,7 @@ using InputBuilder =
 
 #define CALLDATA_IMPL_END \
   }                       \
-  ;
+  ;  // NOLINT
 
 CALLDATA_IMPL(ModelInfer, Unary);
 
@@ -363,13 +362,11 @@ CALLDATA_IMPL_END
 template <>
 class InferenceRequestBuilder<CallDataModelInfer*> {
  public:
-  static InferenceRequestPtr build(
-    const CallDataModelInfer* req, size_t& buffer_index,
-    const std::vector<BufferRawPtrs>& input_buffers,
-    std::vector<size_t>& input_offsets,
-    const std::vector<BufferRawPtrs>& output_buffers,
-    std::vector<size_t>& output_offsets, const size_t& batch_size,
-    size_t& batch_offset) {
+  static InferenceRequestPtr build(const CallDataModelInfer* req,
+                                   const BufferRawPtrs& input_buffers,
+                                   std::vector<size_t>& input_offsets,
+                                   const BufferRawPtrs& output_buffers,
+                                   std::vector<size_t>& output_offsets) {
     auto request = std::make_shared<InferenceRequest>();
     const auto& grpc_request = req->getRequest();
 
@@ -379,68 +376,48 @@ class InferenceRequestBuilder<CallDataModelInfer*> {
 
     request->callback_ = nullptr;
 
-    auto buffer_index_backup = buffer_index;
-    auto batch_offset_backup = batch_offset;
-
     for (const auto& input : grpc_request.inputs()) {
-      auto buffers = input_buffers[buffer_index];
+      const auto& buffers = input_buffers;
+      auto index = 0;
       for (auto& buffer : buffers) {
-        auto& offset = input_offsets[buffer_index];
+        auto& offset = input_offsets[index];
 
         request->inputs_.push_back(InputBuilder::build(input, buffer, offset));
         const auto& last_input = request->inputs_.back();
         offset += (last_input.getSize() * last_input.getDatatype().size());
-      }
-      batch_offset++;
-      if (batch_offset == batch_size) {
-        batch_offset = 0;
-        buffer_index++;
-        // std::fill(input_offsets.begin(), input_offsets.end(), 0);
+        index++;
       }
     }
 
     // TODO(varunsh): output_offset is currently ignored! The size of the
     // output needs to come from the worker but we have no such information.
-    buffer_index = buffer_index_backup;
-    batch_offset = batch_offset_backup;
-
     if (grpc_request.outputs_size() != 0) {
       for (const auto& output : grpc_request.outputs()) {
         // TODO(varunsh): we're ignoring incoming output data
         (void)output;
-        auto buffers = output_buffers[buffer_index];
+        const auto& buffers = output_buffers;
+        auto index = 0;
         for (auto& buffer : buffers) {
-          auto& offset = output_offsets[buffer_index];
+          auto& offset = output_offsets[index];
 
           request->outputs_.emplace_back();
           request->outputs_.back().setData(
             static_cast<std::byte*>(buffer->data()) + offset);
-        }
-        batch_offset++;
-        if (batch_offset == batch_size) {
-          batch_offset = 0;
-          buffer_index++;
-          std::fill(output_offsets.begin(), output_offsets.end(), 0);
+          index++;
         }
       }
     } else {
       for (const auto& input : grpc_request.inputs()) {
         (void)input;  // suppress unused variable warning
-        auto buffers = output_buffers[buffer_index];
+        const auto& buffers = output_buffers;
         for (size_t j = 0; j < buffers.size(); j++) {
           auto& buffer = buffers[j];
-          const auto& offset = output_offsets[buffer_index];
+          const auto& offset = output_offsets[j];
 
           request->outputs_.emplace_back();
           request->outputs_.back().setData(
             static_cast<std::byte*>(buffer->data()) + offset);
         }
-        batch_offset++;
-        if (batch_offset == batch_size) {
-          batch_offset = 0;
-          buffer_index++;
-          std::fill(output_offsets.begin(), output_offsets.end(), 0);
-        }
       }
     }
 
@@ -491,18 +468,16 @@ class GrpcApiUnary : public Interface {
   }
 
   std::shared_ptr<InferenceRequest> getRequest(
-    size_t& buffer_index, const std::vector<BufferRawPtrs>& input_buffers,
-    std::vector<size_t>& input_offsets,
-    const std::vector<BufferRawPtrs>& output_buffers,
-    std::vector<size_t>& output_offsets, const size_t& batch_size,
-    size_t& batch_offset) override {
+    const BufferRawPtrs& input_buffers, std::vector<size_t>& input_offsets,
+    const BufferRawPtrs& output_buffers,
+    std::vector<size_t>& output_offsets) override {
 #ifdef PROTEUS_ENABLE_LOGGING
     const auto& logger = this->getLogger();
 #endif
     try {
-      auto request = RequestBuilder::build(
-        this->calldata_, buffer_index, input_buffers, input_offsets,
-        output_buffers, output_offsets, batch_size, batch_offset);
+      auto request =
+        RequestBuilder::build(this->calldata_, input_buffers, input_offsets,
+                              output_buffers, output_offsets);
       Callback callback =
         std::bind(grpcUnaryCallback, this->calldata_, std::placeholders::_1);
       request->setCallback(std::move(callback));
@@ -769,7 +744,6 @@ class GrpcServer final {
   inference::GRPCInferenceService::AsyncService service_;
   std::unique_ptr<::grpc::Server> server_;
   std::vector<std::thread> threads_;
-  ModelRepository repository_;
 };
 
 namespace grpc {
diff --git a/src/proteus/servers/grpc_server.hpp b/src/proteus/servers/grpc_server.hpp
index 49d098a2a..e2b784764 100644
--- a/src/proteus/servers/grpc_server.hpp
+++ b/src/proteus/servers/grpc_server.hpp
@@ -24,8 +24,6 @@
 
 #ifdef PROTEUS_ENABLE_GRPC
 
-#include <string>
-
 namespace proteus::grpc {
 
 void start(int port);
diff --git a/src/proteus/servers/http_server.cpp b/src/proteus/servers/http_server.cpp
index f96682dba..d7b46d0fd 100644
--- a/src/proteus/servers/http_server.cpp
+++ b/src/proteus/servers/http_server.cpp
@@ -24,29 +24,23 @@
 #include <json/value.h>               // for Value, arrayValue
 #include <trantor/utils/Logger.h>     // for Logger, Logger::kWarn
 
-#include <algorithm>  // for transform
-#include <chrono>     // for high_resolution_clock
-#include <exception>  // for exception
-#include <memory>     // for allocator, shared_ptr
-#include <set>        // for set
-#include <stdexcept>  // for invalid_argument
-#include <string>     // for operator+, string
-#include <utility>    // for move
-#include <vector>     // for vector
-
-#include "proteus/batching/batcher.hpp"           // for Batcher
+#include <chrono>         // for high_resolution_clock
+#include <memory>         // for shared_ptr, __share...
+#include <string>         // for allocator, operator+
+#include <unordered_set>  // for unordered_set
+#include <utility>        // for move
+#include <vector>         // for vector
+
 #include "proteus/build_options.hpp"              // for PROTEUS_ENABLE_TRACING
-#include "proteus/clients/http_internal.hpp"      // for propagate, DrogonHttp
-#include "proteus/clients/native.hpp"             // for getHardware
-#include "proteus/core/api.hpp"                   // for modelLoad
-#include "proteus/core/model_repository.hpp"      // for loadModel
+#include "proteus/clients/http_internal.hpp"      // for propagate, errorHtt...
+#include "proteus/core/api.hpp"                   // for hasHardware, modelI...
+#include "proteus/core/exceptions.hpp"            // for runtime_error, inva...
+#include "proteus/core/interface.hpp"             // for Interface
 #include "proteus/core/predict_api_internal.hpp"  // for RequestParametersPtr
-#include "proteus/core/worker_info.hpp"           // for WorkerInfo
 #include "proteus/helpers/string.hpp"             // for toLower
-#include "proteus/observation/logging.hpp"        // for Logger
+#include "proteus/observation/logging.hpp"        // for Logger, PROTEUS_LOG...
 #include "proteus/observation/metrics.hpp"        // for Metrics, MetricCoun...
 #include "proteus/observation/tracing.hpp"        // for startTrace, Trace
-#include "proteus/servers/server.hpp"             // for getLogDirectory
 #include "proteus/servers/websocket_server.hpp"   // for WebsocketServer
 
 using drogon::HttpRequestPtr;
diff --git a/src/proteus/servers/server.cpp b/src/proteus/servers/server.cpp
index b237099d9..2ff1e8abe 100644
--- a/src/proteus/servers/server.cpp
+++ b/src/proteus/servers/server.cpp
@@ -14,16 +14,17 @@
 
 #include "proteus/servers/server.hpp"
 
-#include <thread>
-
-#include "proteus/build_options.hpp"
-#include "proteus/core/manager.hpp"
-#include "proteus/core/model_repository.hpp"
-#include "proteus/core/worker_info.hpp"
-#include "proteus/observation/logging.hpp"
-#include "proteus/observation/tracing.hpp"
-#include "proteus/servers/grpc_server.hpp"
-#include "proteus/servers/http_server.hpp"
+#include <cstdlib>  // for getenv
+#include <string>   // for operator+, string
+#include <thread>   // for thread
+
+#include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_HTTP
+#include "proteus/core/manager.hpp"           // for Manager
+#include "proteus/core/model_repository.hpp"  // for ModelRepository
+#include "proteus/observation/logging.hpp"    // for initLogger, getLogDirec...
+#include "proteus/observation/tracing.hpp"    // for startTracer, stopTracer
+#include "proteus/servers/grpc_server.hpp"    // for start, stop
+#include "proteus/servers/http_server.hpp"    // for stop, start
 
 #ifdef PROTEUS_ENABLE_AKS
 #include <aks/AksSysManagerExt.h>  // for SysManagerExt
diff --git a/src/proteus/servers/websocket_server.cpp b/src/proteus/servers/websocket_server.cpp
index e96b85bd1..52837e0ab 100644
--- a/src/proteus/servers/websocket_server.cpp
+++ b/src/proteus/servers/websocket_server.cpp
@@ -17,14 +17,15 @@
 #include <json/reader.h>  // for CharReader, CharReaderBui...
 #include <json/value.h>   // for Value, arrayValue
 
-#include <algorithm>   // for transform
-#include <functional>  // for _Bind_helper<>::type, _Pl...
-#include <memory>      // for allocator, shared_ptr
-#include <string>      // for string, operator+, char_t...
-#include <utility>     // for move
-
-#include "proteus/batching/batcher.hpp"  // for Batcher
-#include "proteus/clients/http_internal.hpp"
+#include <algorithm>  // for transform
+#include <cctype>     // for tolower
+#include <memory>     // for allocator, shared_ptr
+#include <string>     // for string, operator+, char_t...
+#include <utility>    // for move
+
+#include "proteus/batching/batcher.hpp"           // for Batcher
+#include "proteus/clients/http_internal.hpp"      // for RequestBuilder
+#include "proteus/core/exceptions.hpp"            // for invalid_argument
 #include "proteus/core/manager.hpp"               // for Manager
 #include "proteus/core/predict_api_internal.hpp"  // for RequestParametersPtr
 #include "proteus/core/worker_info.hpp"           // for WorkerInfo
@@ -128,19 +129,14 @@ size_t DrogonWs::getInputSize() {
 }
 
 std::shared_ptr<InferenceRequest> DrogonWs::getRequest(
-  size_t &buffer_index, const std::vector<BufferRawPtrs> &input_buffers,
-  std::vector<size_t> &input_offset,
-  const std::vector<BufferRawPtrs> &output_buffers,
-  std::vector<size_t> &output_offset, const size_t &batch_size,
-  size_t &batch_offset) {
-  std::shared_ptr<InferenceRequest> request;
+  const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offset,
+  const BufferRawPtrs &output_buffers, std::vector<size_t> &output_offset) {
 #ifdef PROTEUS_ENABLE_LOGGING
   const auto &logger = this->getLogger();
 #endif
   try {
     auto request = RequestBuilder::build(
-      this->json_, buffer_index, input_buffers, input_offset, output_buffers,
-      output_offset, batch_size, batch_offset);
+      this->json_, input_buffers, input_offset, output_buffers, output_offset);
     Callback callback =
       [conn = std::move(this->conn_)](const InferenceResponse &response) {
         auto outputs = response.getOutputs();
diff --git a/src/proteus/servers/websocket_server.hpp b/src/proteus/servers/websocket_server.hpp
index f81c2cc46..6e10d8264 100644
--- a/src/proteus/servers/websocket_server.hpp
+++ b/src/proteus/servers/websocket_server.hpp
@@ -21,8 +21,8 @@
 #include <drogon/WebSocketController.h>  // for WS_PATH_ADD, WS_PATH_LIST...
 
 #include <cstddef>    // for size_t
+#include <exception>  // for exception
 #include <memory>     // for shared_ptr, allocator
-#include <stdexcept>  // for invalid_argument
 #include <string>     // for string
 #include <vector>     // for vector
 
@@ -51,11 +51,9 @@ class DrogonWs : public Interface {
            std::shared_ptr<Json::Value> json);
 
   std::shared_ptr<InferenceRequest> getRequest(
-    size_t &buffer_index, const std::vector<BufferRawPtrs> &input_buffers,
-    std::vector<size_t> &input_offsets,
-    const std::vector<BufferRawPtrs> &output_buffers,
-    std::vector<size_t> &output_offsets, const size_t &batch_size,
-    size_t &batch_offset) override;
+    const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+    const BufferRawPtrs &output_buffers,
+    std::vector<size_t> &output_offsets) override;
 
   size_t getInputSize() override;
   void errorHandler(const std::exception &e) override;
diff --git a/src/proteus/workers/aks.cpp b/src/proteus/workers/aks.cpp
index 6c96a4389..e6a43dc29 100644
--- a/src/proteus/workers/aks.cpp
+++ b/src/proteus/workers/aks.cpp
@@ -26,7 +26,6 @@
 #include <cstdint>                 // for int32_t
 #include <future>                  // for future
 #include <memory>                  // for unique_ptr, allocator
-#include <stdexcept>               // for runtime_error
 #include <string>                  // for string, operator+, char...
 #include <thread>                  // for thread
 #include <utility>                 // for move, pair
@@ -135,7 +134,6 @@ void Aks::doAcquire(RequestParameters* parameters) {
 }
 
 void Aks::doRun(BatchPtrQueue* input_queue) {
-  std::shared_ptr<InferenceRequest> req;
   setThreadName("Aks");
 #ifdef PROTEUS_ENABLE_LOGGING
   const auto& logger = this->getLogger();
@@ -148,10 +146,10 @@ void Aks::doRun(BatchPtrQueue* input_queue) {
       break;
     }
     PROTEUS_LOG_INFO(logger, "Got request in aks");
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto& req = batch->requests->at(j);
+    for (unsigned int j = 0; j < batch->size(); j++) {
+      const auto& req = batch->getRequest(j);
 #ifdef PROTEUS_ENABLE_TRACING
-      auto& trace = batch->traces.at(j);
+      const auto& trace = batch->getTrace(j);
       trace->startSpan("aks");
 #endif
       InferenceResponse resp;
@@ -197,19 +195,16 @@ void Aks::doRun(BatchPtrQueue* input_queue) {
 
 #ifdef PROTEUS_ENABLE_METRICS
       auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-        std::chrono::high_resolution_clock::now() - batch->start_times[j]);
+        std::chrono::high_resolution_clock::now() - batch->getTime(j));
       Metrics::getInstance().observeSummary(MetricSummaryIDs::kRequestLatency,
                                             duration.count());
 #endif
 #ifdef PROTEUS_ENABLE_TRACING
-      auto context = batch->traces.at(j)->propagate();
+      auto context = trace->propagate();
       resp.setContext(std::move(context));
 #endif
       req->runCallbackOnce(resp);
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
   PROTEUS_LOG_INFO(logger, "Aks ending");
 }
diff --git a/src/proteus/workers/aks_detect.cpp b/src/proteus/workers/aks_detect.cpp
index 46b9c2cde..8025ca07a 100644
--- a/src/proteus/workers/aks_detect.cpp
+++ b/src/proteus/workers/aks_detect.cpp
@@ -152,7 +152,6 @@ void AksDetect::doAcquire(RequestParameters* parameters) {
 }
 
 void AksDetect::doRun(BatchPtrQueue* input_queue) {
-  std::shared_ptr<InferenceRequest> req;
   setThreadName("AksDetect");
 #ifdef PROTEUS_ENABLE_LOGGING
   const auto& logger = this->getLogger();
@@ -166,28 +165,16 @@ void AksDetect::doRun(BatchPtrQueue* input_queue) {
     }
     PROTEUS_LOG_INFO(logger, "Got request in AksDetect");
     std::vector<InferenceResponse> responses;
-    responses.reserve(batch->requests->size());
+    responses.reserve(batch->input_size());
 
     std::vector<std::unique_ptr<vart::TensorBuffer>> v;
-    auto batches = 1;
-    if (batches != 1) {
-      /*
-        While the KServe spec allows any number of input tensors in the
-        request, AKS works most easily when a particular request has 1 batch
-        size worth of data. Supporting more data complicates this code and so
-        for now, we exclude this case.
-      */
-      req->runCallbackOnce(InferenceResponse());
-      continue;
-    }
-    v.reserve(batches);
+    v.reserve(batch->size());
 
     size_t tensor_count = 0;
-    // for (auto& req : *(batch->requests)) {
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto& req = batch->requests->at(j);
+    for (unsigned int j = 0; j < batch->size(); j++) {
+      const auto& req = batch->getRequest(j);
 #ifdef PROTEUS_ENABLE_TRACING
-      auto& trace = batch->traces.at(j);
+      const auto& trace = batch->getTrace(j);
       trace->startSpan("AksDetect");
 #endif
       auto& resp = responses.emplace_back();
@@ -281,8 +268,8 @@ void AksDetect::doRun(BatchPtrQueue* input_queue) {
     }
 
     tensor_count = 0;
-    for (unsigned int k = 0; k < batch->requests->size(); k++) {
-      auto req = (*batch->requests)[k];
+    for (unsigned int k = 0; k < batch->size(); k++) {
+      const auto& req = batch->getRequest(k);
       auto inputs = req->getInputs();
       auto outputs = req->getOutputs();
       auto& resp = responses[k];
@@ -311,20 +298,18 @@ void AksDetect::doRun(BatchPtrQueue* input_queue) {
 
 #ifdef PROTEUS_ENABLE_METRICS
       auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-        std::chrono::high_resolution_clock::now() - batch->start_times[k]);
+        std::chrono::high_resolution_clock::now() - batch->getTime(k));
       Metrics::getInstance().observeSummary(MetricSummaryIDs::kRequestLatency,
                                             duration.count());
 #endif
 
 #ifdef PROTEUS_ENABLE_TRACING
-      auto context = batch->traces.at(k)->propagate();
+      const auto& trace = batch->getTrace(k);
+      auto context = trace->propagate();
       resp.setContext(std::move(context));
 #endif
       req->runCallbackOnce(resp);
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
   PROTEUS_LOG_INFO(logger, "AksDetect ending");
 }
diff --git a/src/proteus/workers/aks_detect_stream.cpp b/src/proteus/workers/aks_detect_stream.cpp
index b72982fb4..14d656afa 100644
--- a/src/proteus/workers/aks_detect_stream.cpp
+++ b/src/proteus/workers/aks_detect_stream.cpp
@@ -147,7 +147,6 @@ void AksDetectStream::doAcquire(RequestParameters* parameters) {
 }
 
 void AksDetectStream::doRun(BatchPtrQueue* input_queue) {
-  std::shared_ptr<InferenceRequest> req;
   setThreadName("AksDetectStream");
 #ifdef PROTEUS_ENABLE_LOGGING
   const auto& logger = this->getLogger();
@@ -161,10 +160,10 @@ void AksDetectStream::doRun(BatchPtrQueue* input_queue) {
     }
 
     PROTEUS_LOG_INFO(logger, "Got request in AksDetectStream");
-    for (unsigned int k = 0; k < batch->requests->size(); k++) {
-      auto& req = batch->requests->at(k);
+    for (unsigned int k = 0; k < batch->size(); k++) {
+      const auto& req = batch->getRequest(k);
 #ifdef PROTEUS_ENABLE_TRACING
-      auto& trace = batch->traces.at(k);
+      const auto& trace = batch->getTrace(k);
       trace->startSpan("aks_detect_stream");
 #endif
       auto inputs = req->getInputs();
@@ -350,9 +349,6 @@ void AksDetectStream::doRun(BatchPtrQueue* input_queue) {
         }
       }
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
   PROTEUS_LOG_INFO(logger, "AksDetectStream ending");
 }
diff --git a/src/proteus/workers/echo.cpp b/src/proteus/workers/echo.cpp
index 67268040c..d38ec9062 100644
--- a/src/proteus/workers/echo.cpp
+++ b/src/proteus/workers/echo.cpp
@@ -112,7 +112,6 @@ void Echo::doAcquire(RequestParameters* parameters) {
 }
 
 void Echo::doRun(BatchPtrQueue* input_queue) {
-  std::shared_ptr<InferenceRequest> req;
   setThreadName("Echo");
 #ifdef PROTEUS_ENABLE_LOGGING
   const auto& logger = this->getLogger();
@@ -129,10 +128,10 @@ void Echo::doRun(BatchPtrQueue* input_queue) {
     Metrics::getInstance().incrementCounter(
       MetricCounterIDs::kPipelineIngressWorker);
 #endif
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto& req = batch->requests->at(j);
+    for (unsigned int j = 0; j < batch->size(); j++) {
+      const auto& req = batch->getRequest(j);
 #ifdef PROTEUS_ENABLE_TRACING
-      auto& trace = batch->traces.at(j);
+      const auto& trace = batch->getTrace(j);
       trace->startSpan("echo");
 #endif
       InferenceResponse resp;
@@ -188,14 +187,11 @@ void Echo::doRun(BatchPtrQueue* input_queue) {
       Metrics::getInstance().incrementCounter(
         MetricCounterIDs::kPipelineEgressWorker);
       auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-        std::chrono::high_resolution_clock::now() - batch->start_times[j]);
+        std::chrono::high_resolution_clock::now() - batch->getTime(j));
       Metrics::getInstance().observeSummary(MetricSummaryIDs::kRequestLatency,
                                             duration.count());
 #endif
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
   PROTEUS_LOG_INFO(logger, "Echo ending");
 }
diff --git a/src/proteus/workers/fake.cpp b/src/proteus/workers/fake.cpp
deleted file mode 100644
index ce0824b23..000000000
--- a/src/proteus/workers/fake.cpp
+++ /dev/null
@@ -1,203 +0,0 @@
-// Copyright 2021 Xilinx Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * @file
- * @brief Implements a fake worker for testing
- */
-
-#include <chrono>   // for chrono_literals
-#include <cstddef>  // for size_t, byte
-#include <cstdint>  // for uint32_t, int32_t
-#include <memory>   // for unique_ptr, allocator
-#include <mutex>    // for mutex, lock_guard
-#include <thread>   // for thread, sleep_for
-#include <utility>  // for move
-#include <vector>   // for vector
-
-#include "proteus/batching/fake_batcher.hpp"  // for FakeBatcher
-#include "proteus/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_TRACING
-#include "proteus/core/data_types.hpp"        // for DataType, DataType::UINT32
-#include "proteus/core/predict_api.hpp"       // for InferenceResponse, Requ...
-#include "proteus/helpers/ctpl.h"             // for thread_pool
-#include "proteus/helpers/declarations.hpp"   // for BufferPtr, InferenceRes...
-#include "proteus/helpers/thread.hpp"         // for setThreadName
-#include "proteus/observation/logging.hpp"    // for Logger
-#include "proteus/observation/tracing.hpp"    // for startFollowSpan, SpanPtr
-#include "proteus/workers/worker.hpp"         // for Worker
-
-// define a duration of time to sleep. May be overidden at compile-time
-#ifndef DURATION
-#define DURATION 1ms
-#endif
-
-using namespace std::chrono_literals;
-
-namespace proteus {
-
-namespace workers {
-
-/**
- * @brief The Fake worker is a worker for testing that does nothing but consume
- * incoming batches with a compile-time sleep duration. It responds with an
- * empty response.
- *
- */
-class Fake : public Worker {
- public:
-  using Worker::Worker;
-  std::thread spawn(BatchPtrQueue* input_queue) override;
-
- private:
-  void doInit(RequestParameters* parameters) override;
-  size_t doAllocate(size_t num) override;
-  void doAcquire(RequestParameters* parameters) override;
-  void doRun(BatchPtrQueue* input_queue) override;
-  void doRelease() override;
-  void doDeallocate() override;
-  void doDestroy() override;
-
-  using Worker::makeBatcher;
-  std::vector<std::unique_ptr<Batcher>> makeBatcher(int num = 1) override {
-    return this->makeBatcher<FakeBatcher>(num);
-  };
-
-  ctpl::thread_pool pool_;
-  std::mutex mutex_;
-};
-
-std::thread Fake::spawn(BatchPtrQueue* input_queue) {
-  return std::thread(&Fake::run, this, input_queue);
-}
-
-void Fake::doInit(RequestParameters* parameters) {
-  constexpr auto kMaxBufferNum = 50;
-
-  auto max_buffer_num = kMaxBufferNum;
-  if (parameters->has("max_buffer_num")) {
-    max_buffer_num = parameters->get<int32_t>("max_buffer_num");
-  }
-  this->max_buffer_num_ = max_buffer_num;
-
-  this->batch_size_ = 4;
-}
-
-size_t Fake::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  VectorBuffer::allocate(this->input_buffers_, buffer_num,
-                         1 * this->batch_size_, DataType::UINT32);
-  VectorBuffer::allocate(this->output_buffers_, buffer_num,
-                         1 * this->batch_size_, DataType::UINT32);
-  return buffer_num;
-}
-
-void Fake::doAcquire(RequestParameters* parameters) {
-  constexpr auto kThreads = 3;
-
-  auto threads = kThreads;
-  if (parameters->has("threads")) {
-    threads = parameters->get<int32_t>("threads");
-  }
-  this->pool_.resize(threads);
-
-  this->metadata_.addInputTensor("input", DataType::UINT32, {1});
-  this->metadata_.addOutputTensor("output", DataType::UINT32, {1});
-}
-
-void Fake::doRun(BatchPtrQueue* input_queue) {
-  std::shared_ptr<InferenceRequest> req;
-  setThreadName("Fake");
-#ifdef PROTEUS_ENABLE_LOGGING
-  const auto& logger = this->getLogger();
-#endif
-
-  while (true) {
-    BatchPtr batch;
-    input_queue->wait_dequeue(batch);
-    if (batch == nullptr) {
-      break;
-    }
-    PROTEUS_LOG_INFO(logger, "Got request in fake");
-    this->pool_.push([this, batch = std::move(batch)](int id) {
-      (void)id;  // suppress unused variable warning
-
-      std::vector<InferenceResponse> responses;
-      responses.reserve(batch->requests->size());
-
-      // int tensor_count = 0;
-      for (unsigned int j = 0; j < batch->requests->size(); j++) {
-        auto& req = batch->requests->at(j);
-#ifdef PROTEUS_ENABLE_TRACING
-        auto& trace = batch->traces.at(j);
-        trace->startSpan("fake");
-#endif
-        auto& resp = responses.emplace_back();
-        resp.setID("");
-        resp.setModel("fake");
-        (void)req;
-      }
-
-      {
-        std::lock_guard<std::mutex> guard(this->mutex_);
-        std::this_thread::sleep_for(DURATION);
-      }
-
-      for (unsigned int k = 0; k < batch->requests->size(); k++) {
-        auto req = (*batch->requests)[k];
-        auto inputs = req->getInputs();
-        auto outputs = req->getOutputs();
-        auto& resp = responses[k];
-
-        auto output =
-          InferenceResponseOutput(nullptr, {1}, DataType::UINT32, "fake");
-        auto buffer = std::make_shared<std::vector<uint32_t>>();
-        buffer->resize(1);
-        (*buffer)[0] = 0;
-        auto my_data_cast = std::reinterpret_pointer_cast<std::byte>(buffer);
-        output.setData(std::move(my_data_cast));
-        resp.addOutput(output);
-
-#ifdef PROTEUS_ENABLE_TRACING
-        auto context = batch->traces.at(k)->propagate();
-        resp.setContext(std::move(context));
-#endif
-
-        req->runCallbackOnce(resp);
-      }
-      this->returnBuffers(std::move(batch->input_buffers),
-                          std::move(batch->output_buffers));
-      PROTEUS_LOG_DEBUG(logger, "Returned buffers");
-    });
-  }
-  PROTEUS_LOG_INFO(logger, "Fake ending");
-}
-
-void Fake::doRelease() {}
-void Fake::doDeallocate() { this->pool_.stop(true); }
-void Fake::doDestroy() {}
-
-}  // namespace workers
-
-}  // namespace proteus
-
-extern "C" {
-// using smart pointer here may cause problems inside shared object so managing
-// manually
-proteus::workers::Worker* getWorker() {
-  return new proteus::workers::Fake("Fake", "CPU");
-}
-}  // extern C
diff --git a/src/proteus/workers/invert_image.cpp b/src/proteus/workers/invert_image.cpp
index 92b222a60..0746f7657 100644
--- a/src/proteus/workers/invert_image.cpp
+++ b/src/proteus/workers/invert_image.cpp
@@ -144,7 +144,6 @@ void InvertImage::doAcquire(RequestParameters* parameters) {
 }
 
 void InvertImage::doRun(BatchPtrQueue* input_queue) {
-  std::shared_ptr<InferenceRequest> req;
   setThreadName("InvertImage");
 #ifdef PROTEUS_ENABLE_LOGGING
   const auto& logger = this->getLogger();
@@ -158,10 +157,10 @@ void InvertImage::doRun(BatchPtrQueue* input_queue) {
     }
 
     PROTEUS_LOG_INFO(logger, "Got request in InvertImage");
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto& req = batch->requests->at(j);
+    for (unsigned int j = 0; j < batch->size(); j++) {
+      const auto& req = batch->getRequest(j);
 #ifdef PROTEUS_ENABLE_TRACING
-      auto& trace = batch->traces.at(j);
+      const auto& trace = batch->getTrace(j);
       trace->startSpan("InvertImage");
 #endif
       InferenceResponse resp;
@@ -242,7 +241,7 @@ void InvertImage::doRun(BatchPtrQueue* input_queue) {
       }
 #ifdef PROTEUS_ENABLE_METRICS
       auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-        std::chrono::high_resolution_clock::now() - batch->start_times[j]);
+        std::chrono::high_resolution_clock::now() - batch->getTime(j));
       Metrics::getInstance().observeSummary(MetricSummaryIDs::kRequestLatency,
                                             duration.count());
 #endif
@@ -252,9 +251,6 @@ void InvertImage::doRun(BatchPtrQueue* input_queue) {
 #endif
       req->runCallbackOnce(resp);
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
   PROTEUS_LOG_INFO(logger, "InvertImage ending");
 }
diff --git a/src/proteus/workers/invert_video.cpp b/src/proteus/workers/invert_video.cpp
index bae4a9346..019eba130 100644
--- a/src/proteus/workers/invert_video.cpp
+++ b/src/proteus/workers/invert_video.cpp
@@ -25,7 +25,6 @@
 #include <opencv2/videoio.hpp>    // for VideoCapture, CV_CAP_PRO...
 #include <string>                 // for string, operator+, char_...
 #include <thread>                 // for thread
-#include <utility>                // for move
 #include <vector>                 // for vector
 
 #include "proteus/batching/batcher.hpp"       // for Batch, BatchPtrQueue
@@ -108,7 +107,6 @@ void InvertVideo::doAcquire(RequestParameters* parameters) {
 }
 
 void InvertVideo::doRun(BatchPtrQueue* input_queue) {
-  std::shared_ptr<InferenceRequest> req;
   setThreadName("InvertVideo");
 #ifdef PROTEUS_ENABLE_LOGGING
   const auto& logger = this->getLogger();
@@ -122,10 +120,10 @@ void InvertVideo::doRun(BatchPtrQueue* input_queue) {
     }
 
     PROTEUS_LOG_INFO(logger, "Got request in InvertVideo");
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto& req = batch->requests->at(j);
+    for (unsigned int j = 0; j < batch->size(); j++) {
+      const auto& req = batch->getRequest(j);
 #ifdef PROTEUS_ENABLE_TRACING
-      auto& trace = batch->traces.at(j);
+      const auto& trace = batch->getTrace(j);
       trace->startSpan("InvertVideo");
 #endif
       auto inputs = req->getInputs();
@@ -193,13 +191,7 @@ void InvertVideo::doRun(BatchPtrQueue* input_queue) {
         }
       }
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
-  // if (req != nullptr && req->getWsConn()->connected()) {
-  //   req->getWsConn()->shutdown(drogon::CloseCode::kNormalClosure);
-  // }
   PROTEUS_LOG_INFO(logger, "InvertVideo ending");
 }
 
diff --git a/src/proteus/workers/migraphx.cpp b/src/proteus/workers/migraphx.cpp
index 64c6a47f3..49ed041b9 100644
--- a/src/proteus/workers/migraphx.cpp
+++ b/src/proteus/workers/migraphx.cpp
@@ -386,15 +386,12 @@ void MIGraphXWorker::doRun(BatchPtrQueue* input_queue) {
 #endif
 
     // Safety check:  count all the items in all the requests in this batch
-    size_t items_in_batch = 0;
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto& req_b = batch->requests->at(j);
-      items_in_batch += req_b->getInputSize();
-    }
-    PROTEUS_LOG_DEBUG(
-      logger, "MIGraphXWorker::doRun received request batch with " +
-                std::to_string(batch->requests->size()) + " requests, " +
-                std::to_string(items_in_batch) + " items");
+    size_t items_in_batch = batch->size();
+
+    PROTEUS_LOG_DEBUG(logger,
+                      "MIGraphXWorker::doRun received request batch with " +
+                        std::to_string(batch->size()) + " requests, " +
+                        std::to_string(items_in_batch) + " items");
     if (items_in_batch > this->batch_size_) {
       PROTEUS_LOG_WARN(logger, "migraphx was passed a batch with " +
                                  std::to_string(items_in_batch) +
@@ -409,7 +406,7 @@ void MIGraphXWorker::doRun(BatchPtrQueue* input_queue) {
 
     // The 0'th data pointer for the 0'th request is the base address of the
     // whole data array for the batch
-    auto& req0 = batch->requests->at(0);
+    auto& req0 = batch->getRequest(0);
 
     // The initial implementation of migraphx worker assumes that the model
     // has only one input tensor (for Resnet models, an image) and identifying
@@ -439,8 +436,7 @@ void MIGraphXWorker::doRun(BatchPtrQueue* input_queue) {
       // param_shapes or names
       auto input = param_shapes.names().front();  // "data"
 
-      params.add(input,
-                 migraphx::argument(param_shapes[input], (void*)input_buffer));
+      params.add(input, migraphx::argument(param_shapes[input], input_buffer));
       //
       // Run the inference
       //
@@ -467,8 +463,8 @@ void MIGraphXWorker::doRun(BatchPtrQueue* input_queue) {
       size_t input_index = 0;
 
       // for each request in the batch
-      for (unsigned int j = 0; j < batch->requests->size(); j++) {
-        auto& req = batch->requests->at(j);
+      for (unsigned int j = 0; j < batch->size(); j++) {
+        auto& req = batch->getRequest(j);
         try {
           InferenceResponse resp;
           resp.setID(req->getID());
@@ -559,7 +555,7 @@ void MIGraphXWorker::doRun(BatchPtrQueue* input_queue) {
           Metrics::getInstance().incrementCounter(
             MetricCounterIDs::kPipelineEgressWorker);
           auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-            std::chrono::high_resolution_clock::now() - batch->start_times[j]);
+            std::chrono::high_resolution_clock::now() - batch->getTime(j));
           Metrics::getInstance().observeSummary(
             MetricSummaryIDs::kRequestLatency, duration.count());
 #endif
@@ -571,29 +567,24 @@ void MIGraphXWorker::doRun(BatchPtrQueue* input_queue) {
           req->runCallbackError(
             std::string("Error processing Migraphx request: ") + e.what());
         }
-
       }  // end j, request
     } catch (const std::exception& e) {
       // This outer catch block catches exceptions in evaluation of the batch.
       PROTEUS_LOG_ERROR(logger, e.what());
       // Pass error message back as reply for each request in the batch
-      for (auto& req_e : *(batch->requests)) {
+      const auto& requests = batch->getRequests();
+      for (auto& req_e : requests) {
         req_e->runCallbackError(std::string("Migraphx inference error: ") +
                                 e.what());
       }
     }
 
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
     auto batch_duration = std::chrono::duration_cast<std::chrono::microseconds>(
       std::chrono::high_resolution_clock::now() - batch_tp);
     PROTEUS_LOG_INFO(
       logger, std::string("Finished migraphx batch processing; batch size: ") +
                 std::to_string(batch_size_) + "  elapsed time: " +
                 std::to_string(batch_duration.count()) + " us");
-
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
-
   }  // end while (batch)
   PROTEUS_LOG_INFO(logger, "Migraphx::doRun ending");
 }
diff --git a/src/proteus/workers/pt_zendnn.cpp b/src/proteus/workers/pt_zendnn.cpp
index 8bfa6045e..2930d66d2 100644
--- a/src/proteus/workers/pt_zendnn.cpp
+++ b/src/proteus/workers/pt_zendnn.cpp
@@ -17,28 +17,36 @@
  * @brief Implements the PtZendnn worker
  */
 
+#include <torch/script.h>  // for IValue, Tensor, Device
+
+#include <algorithm>   // for copy
+#include <chrono>      // for duration, operator-
 #include <cstddef>     // for size_t, byte
-#include <cstdint>     // for uint32_t, int32_t
-#include <filesystem>  // for path
+#include <cstdint>     // for int32_t, uint64_t
+#include <cstring>     // for memcpy
+#include <exception>   // for exception
+#include <filesystem>  // for path, exists, filesystem
+#include <functional>  // for multiplies
 #include <memory>      // for unique_ptr, allocator
-#include <string>      // for string
+#include <numeric>     // for accumulate
+#include <ratio>       // for milli, micro
+#include <string>      // for string, operator+, to_s...
 #include <thread>      // for thread
 #include <utility>     // for move
 #include <vector>      // for vector
 
-#include "proteus/batching/hard.hpp"          // for HardBatcher
+#include "proteus/batching/hard.hpp"          // for Batch, BatchPtrQueue
 #include "proteus/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_TRACING
-#include "proteus/core/data_types.hpp"        // for DataType, DataType::UINT32
-#include "proteus/core/predict_api.hpp"       // for InferenceRequest, Infere...
-#include "proteus/helpers/declarations.hpp"   // for BufferPtr, InferenceResp...
-#include "proteus/helpers/string.hpp"         // for endswith
+#include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_LOGGING
+#include "proteus/core/data_types.hpp"        // for DataType, DataType::FP32
+#include "proteus/core/exceptions.hpp"        // for external_error, file_no...
+#include "proteus/core/predict_api.hpp"       // for InferenceResponse, Requ...
+#include "proteus/helpers/declarations.hpp"   // for InferenceResponseOutput
 #include "proteus/helpers/thread.hpp"         // for setThreadName
-#include "proteus/observation/logging.hpp"    // for Logger
-#include "proteus/observation/metrics.hpp"    // for Metrics
-#include "proteus/observation/tracing.hpp"    // for startFollowSpan, SpanPtr
-#include "proteus/workers/worker.hpp"         // for Worker
-#include "torch/script.h"                     // for PyTorch
+#include "proteus/observation/logging.hpp"    // for Logger, PROTEUS_LOG_INFO
+#include "proteus/observation/metrics.hpp"    // for Metrics, MetricCounterIDs
+#include "proteus/observation/tracing.hpp"    // for Trace
+#include "proteus/workers/worker.hpp"         // for Worker, kNumBufferAuto
 
 namespace fs = std::filesystem;
 
@@ -204,10 +212,10 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
       break;
     }
     PROTEUS_LOG_DEBUG(logger, "Got request in PtZendnn. Size: " +
-                                std::to_string(batch->requests->size()));
+                                std::to_string(batch->size()));
 
     std::vector<InferenceResponse> responses;
-    responses.reserve(batch->requests->size());
+    responses.reserve(batch->size());
 
     // This ensures no gradient is calculated and provide performance boost
     torch::NoGradGuard no_grad;
@@ -215,17 +223,11 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
 
     size_t input_size = 0;
     std::vector<torch::jit::IValue> input_vec;
+    auto tensors = static_cast<int>(batch->size());
 
-    // Find total number of images to initialize for PT tensor
-    unsigned tensor_count = 0;
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto req = batch->requests->at(j);
-      tensor_count += req->getInputs().size();
-    }
     // Initialize a PT tensor with required shape
-    torch::Tensor input_tensor =
-      torch::empty({tensor_count, image_channels_, image_height_, image_width_},
-                   torch::kF32);
+    torch::Tensor input_tensor = torch::empty(
+      {tensors, image_channels_, image_height_, image_width_}, torch::kF32);
 
 #ifdef PROTEUS_ENABLE_METRICS
     Metrics::getInstance().incrementCounter(
@@ -233,11 +235,11 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
 #endif
     size_t vec_size = 0;
     auto TotalStart = std::chrono::high_resolution_clock::now();
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      const auto& req = batch->requests->at(j);
+    for (unsigned int j = 0; j < batch->size(); j++) {
+      const auto& req = batch->getRequest(j);
 
 #ifdef PROTEUS_ENABLE_TRACING
-      const auto& trace = batch->traces.at(j);
+      const auto& trace = batch->getTrace(j);
       trace->startSpan("ptzendnn");
 #endif
       auto& resp = responses.emplace_back();
@@ -255,7 +257,7 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
         auto input_shape = input.getShape();
         input_size = reduce_mult(input_shape);
 
-        auto* floatBuffer = (float*)input_buffer;
+        auto* floatBuffer = static_cast<float*>(input_buffer);
         std::copy(floatBuffer, floatBuffer + input_size,
                   input_tensor.data_ptr<float>() + vec_size);
         vec_size = vec_size + input_size;
@@ -272,7 +274,7 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
       prediction = this->model.forward(input_vec);
     } catch (const c10::Error& e) {
       PROTEUS_LOG_ERROR(logger, "Model not suported/Issue with the model");
-      for (const auto& req : *(batch->requests)) {
+      for (const auto& req : *batch) {
         req->runCallbackError("Something went wrong");
       }
     }
@@ -282,13 +284,13 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
       std::chrono::duration<float, std::milli> duration = stop - start;
       float time_tmp = duration.count();
       PROTEUS_LOG_INFO(logger, "Time taken for " +
-                                 std::to_string(tensor_count) +
+                                 std::to_string(batch->size()) +
                                  " images: " + std::to_string(time_tmp));
     }
     at::Tensor output_tensor;
-    if (!prediction.isTuple())
+    if (!prediction.isTuple()) {
       output_tensor = prediction.toTensor();
-    else {
+    } else {
       // For some models like InceptionV3 and GoogleNet which returns Tuple
       output_tensor = prediction.toTuple()->elements()[0].toTensor();
     }
@@ -296,8 +298,8 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
     // Copy the output from the model to the response object
     size_t response_size = output_classes_;
     std::vector<size_t> new_shape = {response_size};
-    for (unsigned int k = 0; k < batch->requests->size(); k++) {
-      auto req = (*batch->requests)[k];
+    for (unsigned int k = 0; k < batch->size(); k++) {
+      const auto& req = batch->getRequest(k);
       auto inputs = req->getInputs();
       auto outputs = req->getOutputs();
       auto& resp = responses[k];
@@ -325,7 +327,7 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
       }
 
 #ifdef PROTEUS_ENABLE_TRACING
-      auto context = batch->traces.at(k)->propagate();
+      auto context = batch->getTrace(k)->propagate();
       resp.setContext(std::move(context));
 #endif
 
@@ -339,14 +341,11 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
 #ifdef PROTEUS_ENABLE_METRICS
       auto now = std::chrono::high_resolution_clock::now();
       std::chrono::duration<double, std::micro> duration =
-        now - batch->start_times[k];
+        now - batch->getTime(k);
       Metrics::getInstance().observeSummary(MetricSummaryIDs::kRequestLatency,
                                             duration.count());
 #endif
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
   PROTEUS_LOG_INFO(logger, "PtZendnn ending");
 }
diff --git a/src/proteus/workers/resnet50.cpp b/src/proteus/workers/resnet50.cpp
index 06a4890d1..94d608f88 100644
--- a/src/proteus/workers/resnet50.cpp
+++ b/src/proteus/workers/resnet50.cpp
@@ -164,27 +164,16 @@ void ResNet50::doRun(BatchPtrQueue* input_queue) {
     }
     PROTEUS_LOG_INFO(logger, "Got request in Resnet50");
     std::vector<InferenceResponse> responses;
-    responses.reserve(batch->requests->size());
+    responses.reserve(batch->size());
 
     std::vector<std::unique_ptr<vart::TensorBuffer>> v;
-    auto batches = 1;
-    if (batches != 1) {
-      /*
-        While the KServe spec allows any number of input tensors in the
-        request, AKS works most easily when a particular request has 1 batch
-        size worth of data. Supporting more data complicates this code and so
-        for now, we exclude this case.
-      */
-      req->runCallbackOnce(InferenceResponse());
-      continue;
-    }
-    v.reserve(batches);
+    v.reserve(batch->input_size());
 
     size_t tensor_count = 0;
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto& req = batch->requests->at(j);
+    for (unsigned int j = 0; j < batch->size(); j++) {
+      const auto& req = batch->getRequest(j);
 #ifdef PROTEUS_ENABLE_TRACING
-      auto& trace = batch->traces.at(j);
+      auto& trace = batch->getTrace(j);
       trace->startSpan("Resnet50");
 #endif
       auto& resp = responses.emplace_back();
@@ -271,8 +260,8 @@ void ResNet50::doRun(BatchPtrQueue* input_queue) {
     }
 
     tensor_count = 0;
-    for (unsigned int k = 0; k < batch->requests->size(); k++) {
-      auto req = (*batch->requests)[k];
+    for (unsigned int k = 0; k < batch->size(); k++) {
+      const auto& req = batch->getRequest(k);
       auto inputs = req->getInputs();
       auto outputs = req->getOutputs();
       auto& resp = responses[k];
@@ -301,19 +290,16 @@ void ResNet50::doRun(BatchPtrQueue* input_queue) {
 
 #ifdef PROTEUS_ENABLE_METRICS
       auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-        std::chrono::high_resolution_clock::now() - batch->start_times[k]);
+        std::chrono::high_resolution_clock::now() - batch->getTime(k));
       Metrics::getInstance().observeSummary(MetricSummaryIDs::kRequestLatency,
                                             duration.count());
 #endif
 #ifdef PROTEUS_ENABLE_TRACING
-      auto context = batch->traces.at(k)->propagate();
+      auto context = batch->getTrace(k)->propagate();
       resp.setContext(std::move(context));
 #endif
       req->runCallbackOnce(resp);
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
   PROTEUS_LOG_INFO(logger, "ResNet50 ending");
 }
diff --git a/src/proteus/workers/resnet50_stream.cpp b/src/proteus/workers/resnet50_stream.cpp
index c150f81ab..06f8c87f1 100644
--- a/src/proteus/workers/resnet50_stream.cpp
+++ b/src/proteus/workers/resnet50_stream.cpp
@@ -42,6 +42,7 @@
 
 #include "proteus/batching/batcher.hpp"       // for BatchPtr, BatchPtrQueue
 #include "proteus/buffers/vector_buffer.hpp"  // for VectorBuffer
+#include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_LOGGING
 #include "proteus/core/data_types.hpp"        // for DataType, DataType::STRING
 #include "proteus/core/predict_api.hpp"       // for InferenceResponse, Infe...
 #include "proteus/helpers/base64.hpp"         // for base64_encode
@@ -166,7 +167,7 @@ void ResNet50Stream::doRun(BatchPtrQueue* input_queue) {
     }
 
     PROTEUS_LOG_INFO(logger, "Got request in ResNet50Stream");
-    for (auto& req : *(batch->requests)) {
+    for (const auto& req : *batch) {
       auto inputs = req->getInputs();
       auto outputs = req->getOutputs();
       auto key = req->getParameters()->get<std::string>("key");
@@ -312,9 +313,6 @@ void ResNet50Stream::doRun(BatchPtrQueue* input_queue) {
         }
       }
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
   PROTEUS_LOG_INFO(logger, "ResNet50Stream ending");
 }
diff --git a/src/proteus/workers/tf_zendnn.cpp b/src/proteus/workers/tf_zendnn.cpp
index fdc8980a0..141c17653 100644
--- a/src/proteus/workers/tf_zendnn.cpp
+++ b/src/proteus/workers/tf_zendnn.cpp
@@ -17,29 +17,45 @@
  * @brief Implements the TfZendnn worker
  */
 
-#include <cstddef>  // for size_t, byte
-#include <cstdint>  // for uint32_t, int32_t
-#include <memory>   // for unique_ptr, allocator
-#include <string>   // for string
-#include <thread>   // for thread
-#include <utility>  // for move
-#include <vector>   // for vector
-
-#include "proteus/batching/hard.hpp"          // for HardBatcher
+#include <tensorflow/c/c_api.h>                         // for TF_Version
+#include <tensorflow/core/framework/graph.pb.h>         // for GraphDef
+#include <tensorflow/core/framework/tensor.h>           // for Tensor
+#include <tensorflow/core/framework/tensor_shape.h>     // for TensorShape
+#include <tensorflow/core/framework/tensor_shape.pb.h>  // for tensorflow
+#include <tensorflow/core/framework/tensor_types.h>     // for TTypes<>::Flat
+#include <tensorflow/core/framework/types.pb.h>         // for DT_FLOAT
+#include <tensorflow/core/platform/env.h>               // for ReadBinaryProto
+#include <tensorflow/core/platform/status.h>            // for Status
+#include <tensorflow/core/protobuf/config.pb.h>         // for ConfigProto
+#include <tensorflow/core/public/session.h>             // for NewSession
+#include <tensorflow/core/public/session_options.h>     // for SessionOptions
+
+#include <algorithm>   // for copy, max
+#include <chrono>      // for milliseconds
+#include <cstddef>     // for size_t, byte
+#include <cstdint>     // for int32_t, uint...
+#include <cstring>     // for memcpy
+#include <functional>  // for multiplies
+#include <memory>      // for allocator
+#include <numeric>     // for accumulate
+#include <string>      // for string, opera...
+#include <thread>      // for thread
+#include <utility>     // for pair, move
+#include <vector>      // for vector
+
+#include "proteus/batching/hard.hpp"          // for Batch, BatchP...
 #include "proteus/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_TRACING
-#include "proteus/core/data_types.hpp"        // for DataType, DataType::UINT32
-#include "proteus/core/predict_api.hpp"       // for InferenceRequest, Infere...
-#include "proteus/helpers/declarations.hpp"   // for BufferPtr, InferenceResp...
+#include "proteus/build_options.hpp"          // for PROTEUS_ENABL...
+#include "proteus/core/data_types.hpp"        // for DataType, Dat...
+#include "proteus/core/exceptions.hpp"        // for external_error
+#include "proteus/core/predict_api.hpp"       // for InferenceResp...
+#include "proteus/helpers/declarations.hpp"   // for InferenceResp...
 #include "proteus/helpers/string.hpp"         // for endsWith
 #include "proteus/helpers/thread.hpp"         // for setThreadName
-#include "proteus/observation/logging.hpp"    // for Logger
-#include "proteus/observation/metrics.hpp"    // for Metrics
-#include "proteus/observation/tracing.hpp"    // for startFollowSpan, SpanPtr
-#include "proteus/workers/worker.hpp"         // for Worker
-#include "tensorflow/c/c_api.h"               // for TensorFlow version
-#include "tensorflow/core/platform/env.h"     // for TensorFlow environment
-#include "tensorflow/core/public/session.h"   // for TensorFlow sessions
+#include "proteus/observation/logging.hpp"    // for Logger, PROTE...
+#include "proteus/observation/metrics.hpp"    // for Metrics, Metr...
+#include "proteus/observation/tracing.hpp"    // for Trace
+#include "proteus/workers/worker.hpp"         // for Worker, kNumB...
 
 namespace tf = ::tensorflow;
 
@@ -216,23 +232,22 @@ void TfZendnn::doAcquire(RequestParameters* parameters) {
 }
 
 void TfZendnn::doRun(BatchPtrQueue* input_queue) {
-  std::shared_ptr<InferenceRequest> req;
-  std::unique_ptr<Batch> batch;
   setThreadName("TfZendnn");
 #ifdef PROTEUS_ENABLE_LOGGING
   const auto& logger = this->getLogger();
 #endif
 
   while (true) {
+    std::unique_ptr<Batch> batch;
     input_queue->wait_dequeue(batch);
     if (batch == nullptr) {
       break;
     }
     PROTEUS_LOG_DEBUG(logger, "Got request in TfZendnn. Size: " +
-                                std::to_string(batch->requests->size()));
+                                std::to_string(batch->size()));
 
     std::vector<InferenceResponse> responses;
-    responses.reserve(batch->requests->size());
+    responses.reserve(batch->size());
 
 #ifdef PROTEUS_ENABLE_METRICS
     Metrics::getInstance().incrementCounter(
@@ -242,11 +257,7 @@ void TfZendnn::doRun(BatchPtrQueue* input_queue) {
     auto TotalStart = std::chrono::high_resolution_clock::now();
 
     // Find total number of images to initialize for TF tensor
-    unsigned tensor_count = 0;
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto req = batch->requests->at(j);
-      tensor_count += req->getInputs().size();
-    }
+    auto tensor_count = static_cast<int>(batch->size());
 
     // Initialize a TensorFlow tensor with required shape
     tf::Tensor input_tensor(
@@ -256,11 +267,11 @@ void TfZendnn::doRun(BatchPtrQueue* input_queue) {
     uint64_t input_size = image_height_ * image_width_ * image_channels_;
     size_t vec_size = 0;
 
-    for (unsigned int j = 0; j < batch->requests->size(); j++) {
-      auto& req = batch->requests->at(j);
+    for (unsigned int j = 0; j < batch->size(); j++) {
+      auto& req = batch->getRequest(j);
 
 #ifdef PROTEUS_ENABLE_TRACING
-      auto& trace = batch->traces.at(j);
+      auto& trace = batch->getTrace(j);
       trace->startSpan("tfzendnn");
 #endif
       auto& resp = responses.emplace_back();
@@ -275,7 +286,7 @@ void TfZendnn::doRun(BatchPtrQueue* input_queue) {
       // Get all the inputs from the requests and copy to the TensorFlow tensor
       for (auto& input : inputs) {
         auto* input_buffer = input.getData();
-        const float* floatBuffer = (float*)input_buffer;
+        const auto* floatBuffer = static_cast<float*>(input_buffer);
         std::copy(floatBuffer, floatBuffer + input_size,
                   input_tensor.flat<float>().data() + vec_size);
         vec_size = vec_size + input_size;
@@ -304,15 +315,17 @@ void TfZendnn::doRun(BatchPtrQueue* input_queue) {
 
     if (!status_.ok()) {
       PROTEUS_LOG_ERROR(logger, status_.ToString());
-      req->runCallbackError("Issue with prediction");
+      for (const auto& req : *batch) {
+        req->runCallbackError("Issue with prediction");
+      }
     }
     PROTEUS_LOG_DEBUG(logger, output_tensor[0].DebugString());
 
     // Copy the output from the model to the response object
     size_t response_size = output_classes_;
     std::vector<size_t> new_shape = {response_size};
-    for (unsigned int k = 0; k < batch->requests->size(); k++) {
-      auto req = (*batch->requests)[k];
+    for (unsigned int k = 0; k < batch->size(); k++) {
+      const auto& req = batch->getRequest(k);
       auto inputs = req->getInputs();
       auto outputs = req->getOutputs();
       auto& resp = responses[k];
@@ -341,7 +354,7 @@ void TfZendnn::doRun(BatchPtrQueue* input_queue) {
       }
 
 #ifdef PROTEUS_ENABLE_TRACING
-      auto context = batch->traces.at(k)->propagate();
+      auto context = batch->getTrace(k)->propagate();
       resp.setContext(std::move(context));
 #endif
 
@@ -357,14 +370,11 @@ void TfZendnn::doRun(BatchPtrQueue* input_queue) {
 
 #ifdef PROTEUS_ENABLE_METRICS
       auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-        std::chrono::high_resolution_clock::now() - batch->start_times[k]);
+        std::chrono::high_resolution_clock::now() - batch->getTime(k));
       Metrics::getInstance().observeSummary(MetricSummaryIDs::kRequestLatency,
                                             duration.count());
 #endif
     }
-    this->returnBuffers(std::move(batch->input_buffers),
-                        std::move(batch->output_buffers));
-    PROTEUS_LOG_DEBUG(logger, "Returned buffers");
   }
   PROTEUS_LOG_INFO(logger, "TfZendnn ending");
 }
diff --git a/src/proteus/workers/xmodel.cpp b/src/proteus/workers/xmodel.cpp
index cb70be510..b2ae97c1b 100644
--- a/src/proteus/workers/xmodel.cpp
+++ b/src/proteus/workers/xmodel.cpp
@@ -222,7 +222,6 @@ void XModel::doAcquire(RequestParameters* parameters) {
 }
 
 void XModel::doRun(BatchPtrQueue* input_queue) {
-  std::shared_ptr<InferenceRequest> req;
   std::atomic_int32_t pool_size = 0;
   const int max_pool_size = this->pool_.size() * 4;  // 4 is arbitrary
   setThreadName("XModel");
@@ -236,8 +235,8 @@ void XModel::doRun(BatchPtrQueue* input_queue) {
     if (batch == nullptr) {
       break;
     }
-    PROTEUS_LOG_INFO(logger, "Got request in xmodel: " +
-                               std::to_string(batch->requests->size()));
+    PROTEUS_LOG_INFO(logger,
+                     "Got request in xmodel: " + std::to_string(batch->size()));
 #ifdef PROTEUS_ENABLE_METRICS
     Metrics::getInstance().incrementCounter(
       MetricCounterIDs::kPipelineIngressWorker);
@@ -248,57 +247,48 @@ void XModel::doRun(BatchPtrQueue* input_queue) {
     }
     this->pool_.push([this, batch = std::move(batch), &pool_size](int id) {
       (void)id;  // suppress unused variable warning
-#ifdef PROTEUS_ENABLE_LOGGING
-      const auto& logger = this->getLogger();
-#endif
 #ifdef PROTEUS_ENABLE_TRACING
-      for (unsigned int j = 0; j < batch->requests->size(); j++) {
-        auto& trace = batch->traces.at(j);
+      for (unsigned int j = 0; j < batch->size(); j++) {
+        auto& trace = batch->getTrace(j);
         trace->startSpan("xmodel");
       }
 #endif
 
       std::queue<std::pair<uint32_t, int>> futures;
-      std::vector<vart::TensorBuffer*> outputs_ptrs_global;
-      for (size_t i = 0; i < batch->input_buffers->size(); i++) {
-        std::vector<vart::TensorBuffer*> inputsPtr;
-        std::vector<vart::TensorBuffer*> outputsPtr;
-
-        auto& input_buffers = (*(batch->input_buffers))[i];
-        inputsPtr.reserve(input_buffers.size());
-        auto& output_buffers = (*(batch->output_buffers))[i];
-        outputsPtr.reserve(output_buffers.size());
-        outputs_ptrs_global.reserve(outputs_ptrs_global.size() +
-                                    output_buffers.size());
-
-        for (const auto& buffer : input_buffers) {
-          auto* vart = dynamic_cast<VartTensorBuffer*>(buffer.get());
-          inputsPtr.emplace_back(vart->getTensorBuffer());
-        }
-        for (const auto& buffer : output_buffers) {
-          auto* vart = dynamic_cast<VartTensorBuffer*>(buffer.get());
-          outputsPtr.emplace_back(vart->getTensorBuffer());
-          outputs_ptrs_global.emplace_back(vart->getTensorBuffer());
-        }
+      std::vector<vart::TensorBuffer*> inputsPtr;
+      std::vector<vart::TensorBuffer*> outputsPtr;
 
-        // FIXME(varunsh): there's a bug in rt-engine where calling sync_for_*()
-        // functions for DPUCADF8H results in wrong inferences. The bug has been
-        // identified and fixed so this check can be removed once it's live
-        if (this->kernel_ != "DPUCADF8H") {
-          for (auto* input : inputsPtr) {
-            const auto* tensor = input->get_tensor();
-            input->sync_for_write(
-              0, tensor->get_element_num() / (tensor->get_shape())[0]);
-          }
-        }
+      auto& input_buffers = batch->getInputBuffers();
+      inputsPtr.reserve(input_buffers.size());
+      auto& output_buffers = batch->getOutputBuffers();
+      outputsPtr.reserve(output_buffers.size());
+
+      for (const auto& buffer : input_buffers) {
+        auto* vart = dynamic_cast<VartTensorBuffer*>(buffer.get());
+        inputsPtr.emplace_back(vart->getTensorBuffer());
+      }
+      for (const auto& buffer : output_buffers) {
+        auto* vart = dynamic_cast<VartTensorBuffer*>(buffer.get());
+        outputsPtr.emplace_back(vart->getTensorBuffer());
+      }
 
-        futures.push(getRunner()->execute_async(inputsPtr, outputsPtr));
+      // FIXME(varunsh): there's a bug in rt-engine where calling sync_for_*()
+      // functions for DPUCADF8H results in wrong inferences. The bug has been
+      // identified and fixed so this check can be removed once it's live
+      if (this->kernel_ != "DPUCADF8H") {
+        for (auto* input : inputsPtr) {
+          const auto* tensor = input->get_tensor();
+          input->sync_for_write(
+            0, tensor->get_element_num() / (tensor->get_shape())[0]);
+        }
       }
 
+      futures.push(getRunner()->execute_async(inputsPtr, outputsPtr));
+
       std::vector<InferenceResponse> responses;
-      responses.reserve(batch->requests->size());
+      responses.reserve(batch->size());
 
-      for (auto& req : *(batch->requests)) {
+      for (const auto& req : *batch) {
         auto& resp = responses.emplace_back();
         resp.setID(req->getID());
         resp.setModel("xmodel");
@@ -311,25 +301,23 @@ void XModel::doRun(BatchPtrQueue* input_queue) {
       }
 
       if (this->kernel_ != "DPUCADF8H") {
-        for (auto* output : outputs_ptrs_global) {
+        for (auto* output : outputsPtr) {
           const auto* tensor = output->get_tensor();
           output->sync_for_read(
             0, tensor->get_element_num() / (tensor->get_shape())[0]);
         }
       }
 
-      for (unsigned int k = 0; k < batch->requests->size(); k++) {
-        auto req = (*batch->requests)[k];
+      for (unsigned int k = 0; k < batch->size(); k++) {
+        const auto& req = batch->getRequest(k);
         auto inputs = req->getInputs();
         auto outputs = req->getOutputs();
         auto& resp = responses[k];
 
         auto tensor_count = 0U;
-        auto buffer_index = 0;
         for (unsigned int i = 0; i < inputs.size(); i++) {
           // TODO(varunsh): assuming 1 output tensor (per 1 input)!
-          auto* output_index =
-            (*batch->output_buffers)[buffer_index][0]->data();
+          auto* output_index = output_buffers[0]->data();
           InferenceResponseOutput output;
           auto output_tensors = getRunner()->get_output_tensors();
           auto output_shape =
@@ -362,15 +350,10 @@ void XModel::doRun(BatchPtrQueue* input_queue) {
 
           resp.addOutput(output);
           tensor_count++;
-          if (tensor_count == this->batch_size_) {
-            tensor_count = 0;
-            buffer_index++;
-            // std::fill(input_offsets.begin(), input_offsets.end(), 0);
-          }
         }
 
 #ifdef PROTEUS_ENABLE_TRACING
-        auto context = batch->traces.at(k)->propagate();
+        auto context = batch->getTrace(k)->propagate();
         resp.setContext(std::move(context));
 #endif
 
@@ -379,14 +362,11 @@ void XModel::doRun(BatchPtrQueue* input_queue) {
         Metrics::getInstance().incrementCounter(
           MetricCounterIDs::kPipelineEgressWorker);
         auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-          std::chrono::high_resolution_clock::now() - batch->start_times[k]);
+          std::chrono::high_resolution_clock::now() - batch->getTime(k));
         Metrics::getInstance().observeSummary(MetricSummaryIDs::kRequestLatency,
                                               duration.count());
 #endif
       }
-      this->returnBuffers(std::move(batch->input_buffers),
-                          std::move(batch->output_buffers));
-      PROTEUS_LOG_DEBUG(logger, "Returned buffers");
       pool_size--;
     });
   }
diff --git a/tools/.iwyu.json b/tools/.iwyu.json
index 920bbedb5..089a2598a 100644
--- a/tools/.iwyu.json
+++ b/tools/.iwyu.json
@@ -214,5 +214,37 @@
       "<drogon/HttpRequest.h>",
       "public"
     ]
+  },
+  {
+    "symbol": [
+      "drogon::HttpResponsePtr",
+      "private",
+      "<drogon/HttpResponse.h>",
+      "public"
+    ]
+  },
+  {
+    "include": [
+      "@<ATen/.*>",
+      "private",
+      "\"torch/script.h\"",
+      "public"
+    ]
+  },
+  {
+    "include": [
+      "@<c10/.*>",
+      "private",
+      "\"torch/script.h\"",
+      "public"
+    ]
+  },
+  {
+    "include": [
+      "@<torch/.*>",
+      "private",
+      "\"torch/script.h\"",
+      "public"
+    ]
   }
 ]
diff --git a/tools/lint_cache.sh b/tools/lint_cache.sh
new file mode 100755
index 000000000..5363bb8ea
--- /dev/null
+++ b/tools/lint_cache.sh
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+# Copyright 2022 Xilinx Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exit_trap () {
+    local lc="$BASH_COMMAND" rc=$?
+    if [[ $rc != 0 ]]; then
+        echo "Command [$lc] exited with code [$rc]"
+    fi
+}
+
+trap exit_trap EXIT
+
+set -eo pipefail
+
+# turn on extended globbing
+shopt -s extglob
+
+iwyu=0
+tidy=0
+run=0
+check=0
+
+while true
+do
+  case "$1" in
+    --iwyu      ) iwyu=1  ; shift 1 ;;
+    --tidy      ) tidy=1  ; shift 1 ;;
+    --run       ) run=1   ; shift 1 ;;
+    --check     ) check=1 ; shift 1 ;;
+    -h | --help ) usage   ; exit  0 ;;
+    *) break ;;
+  esac
+done
+
+if [[ $run == 0 && $check == 0 ]]; then
+    echo "At least one of --run or --check must be set"
+    exit 1
+fi
+
+if [[ $iwyu == 0 && $tidy == 0 ]]; then
+    echo "At least one of --iwyu or --tidy must be set"
+    exit 1
+fi
+
+run_iwyu() {
+    source_file="$1"
+
+    echo $source_file
+    # strip the root_dir from the source file path
+    relative_path=${source_file##${root_dir}/}
+    cache_entry="$cache_dir/iwyu/$relative_path"
+    new_hash=$(md5sum $source_file | cut -d ' ' -f 1)
+    if [[ -f "$cache_entry" ]]; then
+        old_hash=$(tail -n 1 $cache_entry)
+        if [[ $old_hash == $new_hash ]]; then
+            return
+        fi
+    fi
+    mkdir -p $(dirname $cache_entry)
+    python3 /usr/local/bin/iwyu_tool.py -p ${build_dir} $source_file -- -Xiwyu --mapping_file=$root_dir/tools/.iwyu.json > $cache_entry 2>&1
+    echo $new_hash >> $cache_entry
+}
+
+run_tidy() {
+    source_file="$1"
+
+    echo $source_file
+    # strip the root_dir from the source file path
+    relative_path=${source_file##${root_dir}/}
+    cache_entry="$cache_dir/tidy/$relative_path"
+    new_hash=$(md5sum $source_file | cut -d ' ' -f 1)
+    if [[ -f "$cache_entry" ]]; then
+        old_hash=$(tail -n 1 $cache_entry)
+        if [[ $old_hash == $new_hash ]]; then
+            return
+        fi
+    fi
+    mkdir -p $(dirname $cache_entry)
+    clang-tidy -format-style=file -p=${build_dir} --extra-arg "-DCV_STATIC_ANALYSIS=0" $source_file > $cache_entry 2>&1
+    echo $new_hash >> $cache_entry
+}
+
+run_check() {
+    source_file="$1"
+    linter="$2"
+
+    # strip the root_dir from the source file path
+    relative_path=${source_file##${root_dir}/}
+    cache_entry="$cache_dir/$linter/$relative_path"
+    if [[ -f "$cache_entry" ]]; then
+        if grep $source_file $cache_entry; then
+            cat $cache_entry
+        fi
+    fi
+}
+
+# directories relative to the root to search for source files
+source_directories=(examples include src tests)
+
+# file extensions to search for
+extensions=(".c" ".cpp" ".cc" ".h" ".hpp")
+
+root_dir="$PROTEUS_ROOT"
+if [[ -z "$root_dir" ]]; then
+    echo "PROTEUS_ROOT isn't set in the env"
+    exit 1
+fi
+
+build_dir="$root_dir/build/Debug"
+if [[ ! -d $build_dir ]]; then
+    echo "The debug build directory doesn't exist, run 'proteus build --debug' once"
+    exit 1
+fi
+
+cache_dir="$build_dir/lint_cache"
+mkdir -p "$cache_dir"
+
+source_files=()
+for directory in ${source_directories[@]}; do
+    for extension in ${extensions[@]}; do
+        # $() strips the trailing newline, so use trick to get it back
+        source_files_x=$(find "$root_dir"/${directory} -path "*/build/*" -prune -o -name "*$extension" -print; echo x)
+        source_files+="${source_files_x%x}"
+    done
+done
+
+for source_file in ${source_files[@]}; do
+    if [[ $run == 1 ]]; then
+        if [[ $iwyu == 1 ]]; then
+            run_iwyu $source_file
+        fi
+        if [[ $tidy == 1 ]]; then
+            run_tidy $source_file
+        fi
+    fi
+    if [[ $check == 1 ]]; then
+        if [[ $iwyu == 1 ]]; then
+            run_check $source_file iwyu
+        fi
+        if [[ $tidy == 1 ]]; then
+            run_check $source_file tidy
+        fi
+    fi
+done


Test patch:
diff --git a/tests/api/CMakeLists.txt b/tests/api/CMakeLists.txt
index 982c56ef2..e96397852 100644
--- a/tests/api/CMakeLists.txt
+++ b/tests/api/CMakeLists.txt
@@ -15,6 +15,7 @@
 list(
   APPEND targets
          test_model_infer
+         test_model_infer_async
          test_model_list
          test_model_ready
          test_server_live
diff --git a/tests/api/test_model_infer_async.cpp b/tests/api/test_model_infer_async.cpp
new file mode 100644
index 000000000..33e4d0326
--- /dev/null
+++ b/tests/api/test_model_infer_async.cpp
@@ -0,0 +1,77 @@
+// Copyright 2022 Xilinx Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cstdint>  // for uint8_t, uint64_t, uin...
+#include <memory>   // for allocator, unique_ptr
+#include <queue>    // for queue
+#include <vector>   // for vector
+
+#include "proteus/proteus.hpp"                 // for InferenceResponse, Grp...
+#include "proteus/testing/gtest_fixtures.hpp"  // for GrpcFixture
+
+void test(proteus::Client* client) {
+  auto endpoint = client->workerLoad("echo", nullptr);
+  EXPECT_EQ(endpoint, "echo");
+
+  std::vector<uint32_t> imgData;
+  auto shape = {1UL};
+  auto size = 1;
+  imgData.reserve(size);
+  imgData.push_back(1);
+
+  proteus::InferenceRequest request;
+  request.addInputTensor(static_cast<void*>(imgData.data()), shape,
+                         proteus::DataType::UINT32);
+
+  const auto num_requests = 16;
+  std::queue<proteus::InferenceResponseFuture> q;
+  for (auto i = 0; i < num_requests; ++i) {
+    q.push(client->modelInferAsync(endpoint, request));
+  }
+
+  for (auto i = 0; i < num_requests; ++i) {
+    auto& future = q.front();
+    auto response = future.get();
+    q.pop();
+
+    EXPECT_FALSE(response.isError());
+    EXPECT_EQ(response.getID(), "");
+    EXPECT_EQ(response.getModel(), "echo");
+
+    auto outputs = response.getOutputs();
+    EXPECT_EQ(outputs.size(), 1);
+    for (const auto& output : outputs) {
+      auto* data = static_cast<std::vector<uint32_t>*>(output.getData());
+      EXPECT_EQ(data->size(), 1);
+      EXPECT_EQ((*data)[0], 2);
+    }
+  }
+
+  client->modelUnload(endpoint);
+}
+
+#ifdef PROTEUS_ENABLE_GRPC
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST_F(GrpcFixture, ModelInfer) { test(client_.get()); }
+#endif
+
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST_F(BaseFixture, ModelInfer) {
+  proteus::NativeClient client;
+  test(&client);
+}
+
+#ifdef PROTEUS_ENABLE_HTTP
+TEST_F(HttpFixture, ModelInfer) { test(client_.get()); }
+#endif
diff --git a/tests/api/test_model_list.cpp b/tests/api/test_model_list.cpp
index 6b88322df..27c78afec 100644
--- a/tests/api/test_model_list.cpp
+++ b/tests/api/test_model_list.cpp
@@ -13,12 +13,14 @@
 // limitations under the License.
 
 #include <algorithm>  // for find
+#include <chrono>     // for milliseconds
 #include <memory>     // for allocator, unique_ptr
-#include <stdexcept>  // for runtime_error
-#include <string>     // for string
+#include <string>     // for string, basic_string
+#include <thread>     // for sleep_for
+#include <vector>     // for vector
 
-#include "proteus/proteus.hpp"                 // for GrpcClient, NativeClient
-#include "proteus/testing/gtest_fixtures.hpp"  // for AssertionResult, Suite...
+#include "proteus/proteus.hpp"                 // for NativeClient, GrpcClient
+#include "proteus/testing/gtest_fixtures.hpp"  // for AssertionResult, Message
 
 void test(proteus::Client* client) {
   const std::string worker = "echo";
diff --git a/tests/api/test_model_load.cpp b/tests/api/test_model_load.cpp
index 36dc3978a..17b64c0e1 100644
--- a/tests/api/test_model_load.cpp
+++ b/tests/api/test_model_load.cpp
@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <memory>     // for allocator, unique_ptr
-#include <stdexcept>  // for runtime_error
-#include <string>     // for string
+#include <memory>  // for allocator, unique_ptr
+#include <string>  // for string
+#include <thread>  // for yield
+#include <vector>  // for vector
 
-#include "proteus/proteus.hpp"                 // for GrpcClient, NativeClient
+#include "proteus/proteus.hpp"                 // for NativeClient, GrpcClient
 #include "proteus/testing/gtest_fixtures.hpp"  // for AssertionResult, Suite...
 
 void test(proteus::Client* client) {
diff --git a/tests/api/test_model_metadata.cpp b/tests/api/test_model_metadata.cpp
index 514b24078..7a7dbb4ec 100644
--- a/tests/api/test_model_metadata.cpp
+++ b/tests/api/test_model_metadata.cpp
@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <chrono>
-#include <string>  // for string
-#include <thread>
+#include <chrono>  // for seconds
+#include <memory>  // for unique_ptr
+#include <string>  // for allocator, string
+#include <thread>  // for sleep_for
+#include <vector>  // for vector
 
-#include "proteus/proteus.hpp"                 // for GrpcClient, NativeClient
+#include "proteus/proteus.hpp"                 // for ModelMetadata, NativeC...
 #include "proteus/testing/gtest_fixtures.hpp"  // for AssertionResult, Suite...
 
 void test(proteus::Client* client) {
diff --git a/tests/api/test_model_ready.cpp b/tests/api/test_model_ready.cpp
index 57f3cdb6e..7534fc099 100644
--- a/tests/api/test_model_ready.cpp
+++ b/tests/api/test_model_ready.cpp
@@ -12,12 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <algorithm>  // for find
-#include <memory>     // for allocator, unique_ptr
-#include <stdexcept>  // for runtime_error
-#include <string>     // for string
+#include <chrono>  // for milliseconds
+#include <memory>  // for allocator, unique_ptr
+#include <string>  // for string
+#include <thread>  // for sleep_for, yield
+#include <vector>  // for vector
 
-#include "proteus/proteus.hpp"                 // for GrpcClient, NativeClient
+#include "proteus/proteus.hpp"                 // for NativeClient, GrpcClient
 #include "proteus/testing/gtest_fixtures.hpp"  // for AssertionResult, Suite...
 
 bool isReady(proteus::Client* client, const std::string& endpoint) {
diff --git a/tests/models/test_mnist.cpp b/tests/models/test_mnist.cpp
index 781eef0df..1ab393ffa 100644
--- a/tests/models/test_mnist.cpp
+++ b/tests/models/test_mnist.cpp
@@ -12,13 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <opencv2/core.hpp>
-#include <opencv2/imgcodecs.hpp>
-#include <opencv2/imgproc.hpp>
-
-#include "gtest/gtest.h"        // for Message, AssertionResult, TestPartResult
-#include "proteus/proteus.hpp"  // for InferenceRequestInput, RequestParameters
-#include "proteus/testing/gtest_fixtures.hpp"  // for AssertionResult, Suite...
+#include <cstdlib>                // for getenv
+#include <memory>                 // for allocator, unique_ptr
+#include <opencv2/core.hpp>       // for Mat, CV_32FC3
+#include <opencv2/imgcodecs.hpp>  // for imread
+#include <opencv2/imgproc.hpp>    // for cvtColor, COLOR_BGR2GRAY
+#include <string>                 // for string, operator+
+#include <thread>                 // for yield
+#include <vector>                 // for vector
+
+#include "proteus/proteus.hpp"                 // for InferenceRequestInput
+#include "proteus/testing/gtest_fixtures.hpp"  // for AssertionResult, Message
 
 namespace proteus {
 
diff --git a/tests/performance/batching/perf_soft.cpp b/tests/performance/batching/perf_soft.cpp
index 65bd2827b..f9c42802f 100644
--- a/tests/performance/batching/perf_soft.cpp
+++ b/tests/performance/batching/perf_soft.cpp
@@ -72,9 +72,6 @@ class PerfSoftBatcherFixture
       std::this_thread::sleep_for(std::chrono::microseconds(delay));
       if (valid_read) {
         count++;
-        this->worker_->putInputBuffer(std::move((*(batch->input_buffers))[0]));
-        this->worker_->putOutputBuffer(
-          std::move((*(batch->output_buffers))[0]));
       }
     } while (valid_read);
     return count;
diff --git a/tests/src/proteus/batching/batcher.cpp b/tests/src/proteus/batching/batcher.cpp
index 7915837f4..1fcef864a 100644
--- a/tests/src/proteus/batching/batcher.cpp
+++ b/tests/src/proteus/batching/batcher.cpp
@@ -19,26 +19,23 @@
 
 #include "proteus/batching/batcher.hpp"
 
-#include <chrono>      // for system_clock::time_point
 #include <cstddef>     // for size_t
+#include <exception>   // for exception
 #include <functional>  // for _Bind_helper<>::type
 #include <future>      // for promise
-#include <memory>      // for unique_ptr, shared_ptr
-#include <stdexcept>   // for invalid_argument
-#include <string>      // for operator+
+#include <memory>      // for shared_ptr, unique_ptr
+#include <string>      // for operator+, char_traits
 #include <utility>     // for move
 #include <vector>      // for vector
 
-#include "proteus/buffers/buffer.hpp"         // IWYU pragma: keep
 #include "proteus/build_options.hpp"          // for PROTEUS_ENABLE_TRACING
 #include "proteus/core/fake_predict_api.hpp"  // for FakeInferenceRequest
-#include "proteus/core/interface.hpp"         // for InterfacePtr, Interface
+#include "proteus/core/interface.hpp"         // for Interface
 #include "proteus/core/predict_api.hpp"       // for InferenceResponsePromis...
-#include "proteus/core/worker_info.hpp"       // for WorkerInfo
 #include "proteus/helpers/queue.hpp"          // for BlockingConcurrentQueue
 #include "proteus/helpers/thread.hpp"         // for setThreadName
-#include "proteus/observation/logging.hpp"    // for Logger
-#include "proteus/observation/tracing.hpp"    // for startFollowSpan, SpanPtr
+#include "proteus/observation/logging.hpp"    // for Logger, PROTEUS_LOG_DEBUG
+#include "proteus/observation/tracing.hpp"    // for Trace
 
 // IWYU pragma: no_forward_declare proteus::Buffer
 
@@ -54,11 +51,9 @@ class FakeInterface : public Interface {
   explicit FakeInterface(InferenceRequestInput request);
 
   std::shared_ptr<InferenceRequest> getRequest(
-    size_t &buffer_index, const std::vector<BufferRawPtrs> &input_buffers,
-    std::vector<size_t> &input_offsets,
-    const std::vector<BufferRawPtrs> &output_buffers,
-    std::vector<size_t> &output_offsets, const size_t &batch_size,
-    size_t &batch_offset) override;
+    const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+    const BufferRawPtrs &output_buffers,
+    std::vector<size_t> &output_offsets) override;
 
   size_t getInputSize() override;
   void errorHandler(const std::exception &e) override;
@@ -86,14 +81,11 @@ void fakeCppCallback(const InferenceResponsePromisePtr &promise,
 }
 
 std::shared_ptr<InferenceRequest> FakeInterface::getRequest(
-  size_t &buffer_index, const std::vector<BufferRawPtrs> &input_buffers,
-  std::vector<size_t> &input_offsets,
-  const std::vector<BufferRawPtrs> &output_buffers,
-  std::vector<size_t> &output_offsets, const size_t &batch_size,
-  size_t &batch_offset) {
+  const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+  const BufferRawPtrs &output_buffers, std::vector<size_t> &output_offsets) {
   auto request = std::make_shared<FakeInferenceRequest>(
-    this->request_, buffer_index, input_buffers, input_offsets, output_buffers,
-    output_offsets, batch_size, batch_offset);
+    this->request_, input_buffers, input_offsets, output_buffers,
+    output_offsets);
   Callback callback =
     std::bind(fakeCppCallback, this->promise_, std::placeholders::_1);
   request->setCallback(std::move(callback));
@@ -112,29 +104,13 @@ void Batcher::run(WorkerInfo *worker) {
   bool run = true;
 
   while (run) {
-    auto batch = std::make_unique<Batch>();
-    batch->requests =
-      std::make_unique<std::vector<std::shared_ptr<InferenceRequest>>>();
-    batch->input_buffers = std::make_unique<std::vector<BufferPtrs>>();
-
-    batch->output_buffers = std::make_unique<std::vector<BufferPtrs>>();
-    auto input_buffer = worker->getInputBuffer();
-    std::vector<size_t> input_offset = {0};
-    auto output_buffer = worker->getOutputBuffer();
-    std::vector<size_t> output_offset = {0};
-    size_t batch_size = 0;
+    auto batch = std::make_unique<Batch>(worker);
+    auto input_buffers = batch->getRawInputBuffers();
+    auto output_buffers = batch->getRawOutputBuffers();
+    std::vector<size_t> input_offset(input_buffers.size(), 0);
+    std::vector<size_t> output_offset(output_buffers.size(), 0);
 
     // wait for the first request
-    std::vector<BufferRawPtrs> input_buffers;
-    input_buffers.emplace_back();
-    for (const auto &buffer : input_buffer) {
-      input_buffers.back().push_back(buffer.get());
-    }
-    std::vector<BufferRawPtrs> output_buffers;
-    output_buffers.emplace_back();
-    for (const auto &buffer : output_buffer) {
-      output_buffers.back().push_back(buffer.get());
-    }
     this->input_queue_->wait_dequeue(req);
     PROTEUS_LOG_DEBUG(logger_,
                       "Got initial request of a new batch for " + this->model_);
@@ -147,33 +123,23 @@ void Batcher::run(WorkerInfo *worker) {
     auto trace = req->getTrace();
     trace->startSpan("fake_batcher");
 #endif
-    // auto fake_req = dynamic_cast<FakeInterface*>(req.get());
-    // InferenceRequestPtr new_req;
-    // if(fake_req != nullptr){
     req->getInputSize();  // initialize the req object
-    size_t buffer_index = 0;
-    auto new_req =
-      req->getRequest(buffer_index, input_buffers, input_offset, output_buffers,
-                      output_offset, this->batch_size_, batch_size);
-    // } else {
-    //   new_req = std::make_unique<FakeInferenceRequest>();
-    // }
+    auto new_req = req->getRequest(input_buffers, input_offset, output_buffers,
+                                   output_offset);
 
-    batch->requests->push_back(new_req);
+    batch->addRequest(new_req);
 
 #ifdef PROTEUS_ENABLE_TRACING
     trace->endSpan();
-    batch->traces.emplace_back(std::move(trace));
+    batch->addTrace(std::move(trace));
 #endif
 #ifdef PROTEUS_ENABLE_METRICS
-    batch->start_times.emplace_back(req->get_time());
+    batch->addTime(req->get_time());
 #endif
     // batch_size += 1;
 
-    if (!batch->requests->empty()) {
+    if (!batch->empty()) {
       PROTEUS_LOG_DEBUG(logger_, "Enqueuing batch for " + this->model_);
-      batch->input_buffers->push_back(std::move(input_buffer));
-      batch->output_buffers->push_back(std::move(output_buffer));
       this->output_queue_->enqueue(std::move(batch));
     }
   }
diff --git a/tests/src/proteus/core/fake_predict_api.hpp b/tests/src/proteus/core/fake_predict_api.hpp
index fcfc2330c..94cad0eeb 100644
--- a/tests/src/proteus/core/fake_predict_api.hpp
+++ b/tests/src/proteus/core/fake_predict_api.hpp
@@ -36,12 +36,11 @@ namespace proteus {
 class FakeInferenceRequest : public InferenceRequest {
  public:
   FakeInferenceRequest();  ///< Constructor
-  FakeInferenceRequest(InferenceRequestInput& req, size_t& buffer_index,
-                       const std::vector<BufferRawPtrs>& input_buffers,
+  FakeInferenceRequest(InferenceRequestInput& req,
+                       const BufferRawPtrs& input_buffers,
                        std::vector<size_t>& input_offsets,
-                       const std::vector<BufferRawPtrs>& output_buffers,
-                       std::vector<size_t>& output_offsets,
-                       const size_t& batch_size, size_t& batch_offset);
+                       const BufferRawPtrs& output_buffers,
+                       std::vector<size_t>& output_offsets);
 };
 
 }  // namespace proteus
diff --git a/tests/src/proteus/core/manager.cpp b/tests/src/proteus/core/manager.cpp
index b105c1e70..f6019cd96 100644
--- a/tests/src/proteus/core/manager.cpp
+++ b/tests/src/proteus/core/manager.cpp
@@ -20,17 +20,16 @@
 
 #include "proteus/core/manager.hpp"
 
-#include <stdexcept>  // for invalid_argument
-#include <thread>     // for thread
-#include <utility>    // for pair, make_pair, move
+#include <thread>   // for thread
+#include <utility>  // for pair, make_pair, move
 
-#include "proteus/build_options.hpp"     // for PROTEUS_ENABLE_LOGGING
+#include "proteus/core/exceptions.hpp"   // for invalid_argument
 #include "proteus/core/worker_info.hpp"  // for WorkerInfo
 #include "proteus/workers/worker.hpp"    // for Worker
 
 namespace proteus {
 
-Manager::Manager() {}
+Manager::Manager() = default;
 
 std::string Manager::loadWorker(std::string const& key,
                                 RequestParameters parameters) {
diff --git a/tests/src/proteus/core/predict_api.cpp b/tests/src/proteus/core/predict_api.cpp
index 8622b1443..c63ac8150 100644
--- a/tests/src/proteus/core/predict_api.cpp
+++ b/tests/src/proteus/core/predict_api.cpp
@@ -17,21 +17,22 @@
  * @brief Implements the fake predict api InferenceRequest
  */
 
-#include <memory>  // for make_unique
-#include <string>  // for string
-#include <vector>  // for vector
+#include "proteus/core/predict_api.hpp"  // for InferenceRequestInput
 
-#include "proteus/core/fake_predict_api.hpp"
+#include <cstddef>  // for size_t
+#include <memory>   // for make_unique
+#include <string>   // for string
+#include <vector>   // for vector
+
+#include "proteus/core/fake_predict_api.hpp"  // for FakeInferenceRequest
+#include "proteus/helpers/declarations.hpp"   // for BufferRawPtrs
 
 namespace proteus {
 
 FakeInferenceRequest::FakeInferenceRequest(
-  InferenceRequestInput& req, size_t& buffer_index,
-  const std::vector<BufferRawPtrs>& input_buffers,
-  std::vector<size_t>& input_offsets,
-  const std::vector<BufferRawPtrs>& output_buffers,
-  std::vector<size_t>& output_offsets, const size_t& batch_size,
-  size_t& batch_offset) {
+  InferenceRequestInput& req, const BufferRawPtrs& input_buffers,
+  std::vector<size_t>& input_offsets, const BufferRawPtrs& output_buffers,
+  std::vector<size_t>& output_offsets) {
   this->id_ = "";
   this->parameters_ = std::make_unique<RequestParameters>();
   this->callback_ = nullptr;
@@ -40,13 +41,10 @@ FakeInferenceRequest::FakeInferenceRequest(
   this->outputs_.emplace_back();
 
   (void)req;
-  (void)buffer_index;
   (void)input_buffers;
   (void)input_offsets;
   (void)output_buffers;
   (void)output_offsets;
-  (void)batch_size;
-  (void)batch_offset;
 }
 
 FakeInferenceRequest::FakeInferenceRequest() {
diff --git a/tests/src/proteus/core/worker_info_buffers_finite.cpp b/tests/src/proteus/core/worker_info_buffers_finite.cpp
index 7df5626b7..ad25de4ad 100644
--- a/tests/src/proteus/core/worker_info_buffers_finite.cpp
+++ b/tests/src/proteus/core/worker_info_buffers_finite.cpp
@@ -27,7 +27,6 @@
 
 #include "proteus/batching/batcher.hpp"      // for Batcher
 #include "proteus/buffers/buffer.hpp"        // IWYU pragma: keep
-#include "proteus/build_options.hpp"         // for PROTEUS_ENABLE_LOGGING
 #include "proteus/core/predict_api.hpp"      // for RequestParameters
 #include "proteus/core/worker_info.hpp"      // for WorkerInfo
 #include "proteus/helpers/declarations.hpp"  // for BufferPtrs
diff --git a/tests/src/proteus/core/worker_info_buffers_infinite.cpp b/tests/src/proteus/core/worker_info_buffers_infinite.cpp
index 46a8a2c6b..a7db1220c 100644
--- a/tests/src/proteus/core/worker_info_buffers_infinite.cpp
+++ b/tests/src/proteus/core/worker_info_buffers_infinite.cpp
@@ -26,7 +26,6 @@
 
 #include "proteus/batching/batcher.hpp"      // for Batcher
 #include "proteus/buffers/buffer.hpp"        // IWYU pragma: keep
-#include "proteus/build_options.hpp"         // for PROTEUS_ENABLE_LOGGING
 #include "proteus/core/predict_api.hpp"      // for RequestParameters
 #include "proteus/core/worker_info.hpp"      // for WorkerInfo
 #include "proteus/helpers/declarations.hpp"  // for BufferPtrs
diff --git a/tests/unit/batching/test_soft.cpp b/tests/unit/batching/test_soft.cpp
index 1ce24cf0f..cd4508aa3 100644
--- a/tests/unit/batching/test_soft.cpp
+++ b/tests/unit/batching/test_soft.cpp
@@ -14,11 +14,12 @@
 
 #include <memory>  // for allocator
 
-#include "gtest/gtest.h"                 // for Test, SuiteApiResolver, TEST
-#include "proteus/batching/soft.hpp"     // for SoftBatcher
-#include "proteus/build_options.hpp"     // for PROTEUS_ENABLE_LOGGING
-#include "proteus/core/interface.hpp"    // IWYU pragma: keep
-#include "proteus/core/worker_info.hpp"  // for WorkerInfo
+#include "gtest/gtest.h"                    // for Test, SuiteApiResolver, TEST
+#include "proteus/batching/soft.hpp"        // for SoftBatcher
+#include "proteus/build_options.hpp"        // for PROTEUS_ENABLE_LOGGING
+#include "proteus/core/interface.hpp"       // IWYU pragma: keep
+#include "proteus/core/worker_info.hpp"     // for WorkerInfo
+#include "proteus/observation/logging.hpp"  // for initLogger, LogLevel, Log...
 
 namespace proteus {
 
diff --git a/tests/unit/batching/test_soft_batching.cpp b/tests/unit/batching/test_soft_batching.cpp
index d1aa14904..128ea92d4 100644
--- a/tests/unit/batching/test_soft_batching.cpp
+++ b/tests/unit/batching/test_soft_batching.cpp
@@ -12,15 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <cstddef>   // for size_t
-#include <cstdint>   // for uint8_t, uint64_t
-#include <memory>    // for allocator, make_unique
-#include <numeric>   // for accumulate
-#include <optional>  // for optional
-#include <utility>   // for pair, move
-#include <vector>    // for vector
-
-#include "gtest/gtest.h"                        // for EXPECT_EQ, UnitTest
+#include <array>             // for array
+#include <cstddef>           // for size_t
+#include <cstdint>           // for uint8_t, uint64_t
+#include <initializer_list>  // for initializer_list
+#include <memory>            // for allocator, make_unique
+#include <optional>          // for optional
+#include <ostream>           // for operator<<, basic_ost...
+#include <utility>           // for move
+#include <vector>            // for vector
+
+#include "gtest/gtest.h"                        // for ParamIteratorInterface
 #include "proteus/batching/soft.hpp"            // for BatchPtr, SoftBatcher
 #include "proteus/buffers/buffer.hpp"           // for Buffer
 #include "proteus/buffers/vector_buffer.hpp"    // for VectorBuffer
@@ -30,6 +32,7 @@
 #include "proteus/core/predict_api.hpp"         // for InferenceRequest, Req...
 #include "proteus/core/worker_info.hpp"         // for WorkerInfo
 #include "proteus/helpers/declarations.hpp"     // for BufferPtrs
+#include "proteus/observation/logging.hpp"      // for initLogger, LogLevel
 
 namespace proteus {
 
@@ -138,9 +141,7 @@ class UnitSoftBatcherFixture : public testing::TestWithParam<BatchConfig> {
         batcher_->getOutputQueue()->wait_dequeue_timed(batch, kTimeoutUs),
         true);
 
-      EXPECT_EQ(batch->input_buffers->size(), 1);
-      EXPECT_EQ(batch->output_buffers->size(), 1);
-      EXPECT_EQ(batch->requests->size(), i);
+      EXPECT_EQ(batch->size(), i);
 
       auto num_tensors = 0;
       for (auto j = tensor_index; j < tensor_index + i; j++) {
@@ -148,15 +149,16 @@ class UnitSoftBatcherFixture : public testing::TestWithParam<BatchConfig> {
       }
       tensor_index += i;
 
-      for (const auto& buffer : *(batch->input_buffers)) {
-        EXPECT_EQ(buffer.size(), 1);
-        auto& first_buffer = buffer[0];
+      const auto& buffers = batch->getInputBuffers();
+      EXPECT_EQ(buffers.size(), 1);
+      for (const auto& buffer : buffers) {
         for (auto j = 0; j < num_tensors; j++) {
-          compare_data(first_buffer.get(), j * data_size_);
+          compare_data(buffer.get(), j * data_size_);
         }
       }
 
-      for (const auto& req : *(batch->requests)) {
+      const auto& reqs = batch->getRequests();
+      for (const auto& req : reqs) {
         req->runCallback(InferenceResponse());
       }
     }
@@ -166,12 +168,10 @@ class UnitSoftBatcherFixture : public testing::TestWithParam<BatchConfig> {
     batcher_->enqueue(std::move(req));
   }
 
-  InferenceRequest create_request(int tensors) {
+  InferenceRequest create_request() {
     InferenceRequest request;
-    for (auto i = 0; i < tensors; ++i) {
-      request.addInputTensor(static_cast<void*>(data_.data()), data_shape_,
-                             DataType::UINT8);
-    }
+    request.addInputTensor(static_cast<void*>(data_.data()), data_shape_,
+                           DataType::UINT8);
     return request;
   }
 
@@ -189,9 +189,11 @@ TEST_P(UnitSoftBatcherFixture, BasicBatching) {
   auto requests = batch_config.requests;
 
   for (const auto& i : requests) {
-    auto request = create_request(i);
-    auto req = std::make_unique<CppNativeApi>(request);
-    this->enqueue(std::move(req));
+    for (auto j = 0; j < i; ++j) {
+      auto request = create_request();
+      auto req = std::make_unique<CppNativeApi>(request);
+      this->enqueue(std::move(req));
+    }
   }
 
   this->check_batch();
@@ -213,8 +215,8 @@ TEST_P(UnitSoftBatcherFixture, BasicBatching) {
 const std::array<BatchConfig, 7> configs = {
   BatchConfig{1, {1}, {1}},          BatchConfig{1, {1, 1}, {1, 1}},
   BatchConfig{2, {1}, {1}},          BatchConfig{2, {1, 1}, {2}},
-  BatchConfig{2, {1, 1, 1}, {2, 1}}, BatchConfig{4, {3, 2}, {1, 1}},
-  BatchConfig{4, {2, 2}, {2}},
+  BatchConfig{2, {1, 1, 1}, {2, 1}}, BatchConfig{4, {1, 1, 1, 1, 1}, {4, 1}},
+  BatchConfig{4, {1, 1, 1, 1}, {4}},
 };
 INSTANTIATE_TEST_SUITE_P(Datatypes, UnitSoftBatcherFixture,
                          testing::ValuesIn(configs));
diff --git a/tests/workers/facedetect.hpp b/tests/workers/facedetect.hpp
index f7767592b..b87457238 100644
--- a/tests/workers/facedetect.hpp
+++ b/tests/workers/facedetect.hpp
@@ -41,6 +41,7 @@ std::vector<char> imgData;
 
 void enqueue(std::vector<std::string>& image_paths, int start_index, int count,
              const std::string& workerName, FutureQueue& my_queue) {
+  proteus::NativeClient client;
   for (int i = 0; i < count; i++) {
     auto img = cv::imread(image_paths[start_index + i]);
     auto shape = {static_cast<uint64_t>(img.size[0]),
@@ -54,8 +55,7 @@ void enqueue(std::vector<std::string>& image_paths, int start_index, int count,
     request.addInputTensor(static_cast<void*>(imgData.data()), shape,
                            proteus::DataType::UINT8);
 
-    auto future =
-      proteus::NativeClient::enqueue(workerName, std::move(request));
+    auto future = client.modelInferAsync(workerName, request);
     my_queue.enqueue(std::move(future));
   }
 }
diff --git a/tests/workers/test_echo.cpp b/tests/workers/test_echo.cpp
index eeb7d930b..8adc13fe9 100644
--- a/tests/workers/test_echo.cpp
+++ b/tests/workers/test_echo.cpp
@@ -12,15 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <cstdint>    // for uint32_t
-#include <future>     // for future
-#include <memory>     // for allocator, make_shared, __shared_ptr_...
-#include <stdexcept>  // for runtime_error
-#include <tuple>      // for tuple, get
-#include <vector>     // for vector
+#include <chrono>   // for seconds
+#include <cstdint>  // for uint32_t
+#include <memory>   // for allocator, make_shared, __shared_ptr_...
+#include <thread>   // for sleep_for
+#include <tuple>    // for tuple, get
+#include <vector>   // for vector
 
-#include "gtest/gtest.h"        // for Message, AssertionResult, TestPartResult
-#include "proteus/proteus.hpp"  // for InferenceRequestInput, RequestParameters
+#include "gtest/gtest.h"        // for ParamIteratorInterface, Message, Test...
+#include "proteus/proteus.hpp"  // for RequestParameters, InferenceRequestInput
 
 namespace proteus {
 
@@ -42,21 +42,9 @@ class EchoParamFixture
       input_0.setParameters(parameters);
     }
 
-    InferenceRequestInput input_1;
-    input_1.setName("echo");
-    input_1.setDatatype(DataType::UINT32);
-    input_1.setShape({1});
-    input_1.setData((void*)(&(inputs_[1])));
-    if (add_input_parameters) {
-      auto parameters = std::make_shared<RequestParameters>();
-      parameters->put("key_1", 1);
-      input_1.setParameters(parameters);
-    }
-
     InferenceRequest request;
     for (auto i = 0; i < multiplier; i++) {
       request.addInputTensor(input_0);
-      request.addInputTensor(input_1);
     }
 
     if (add_outputs) {
@@ -69,7 +57,6 @@ class EchoParamFixture
       }
       for (auto i = 0; i < multiplier; i++) {
         request.addOutputTensor(output);
-        request.addOutputTensor(output);
       }
     }
 
@@ -95,21 +82,19 @@ class EchoParamFixture
     EXPECT_FALSE(response.isError());
     EXPECT_EQ(response.getModel(), "echo");
     auto outputs = response.getOutputs();
-    EXPECT_EQ(outputs.size(), multiplier * 2);
+    EXPECT_EQ(outputs.size(), multiplier);
 
     for (auto i = 0; i < multiplier; i++) {
-      for (auto j = 0; j < 2; j++) {
-        auto output = outputs[(i * 2 + j)];
-        auto* data = static_cast<std::vector<uint32_t>*>(output.getData());
-        EXPECT_EQ(data->size(), 1);
-        EXPECT_EQ((*data)[0], golden_outputs_[j]);
-        EXPECT_EQ(output.getDatatype(), DataType::UINT32);
-        EXPECT_EQ(output.getName(), "echo");
-        EXPECT_TRUE(output.getParameters()->empty());
-        auto shape = output.getShape();
-        EXPECT_EQ(shape.size(), 1);
-        EXPECT_EQ(shape[0], 1);
-      }
+      const auto& output = outputs[i];
+      auto* data = static_cast<std::vector<uint32_t>*>(output.getData());
+      EXPECT_EQ(data->size(), 1);
+      EXPECT_EQ((*data)[0], golden_outputs_[0]);
+      EXPECT_EQ(output.getDatatype(), DataType::UINT32);
+      EXPECT_EQ(output.getName(), "echo");
+      EXPECT_TRUE(output.getParameters()->empty());
+      auto shape = output.getShape();
+      EXPECT_EQ(shape.size(), 1);
+      EXPECT_EQ(shape[0], 1);
     }
 
     if (add_id) {
@@ -120,14 +105,11 @@ class EchoParamFixture
   proteus::Server server_;
 
  private:
-  const int inputs_[2] = {3, 5};
-  const int golden_outputs_[2] = {4, 6};
+  const int inputs_[1] = {3};
+  const int golden_outputs_[1] = {4};
 };
 
 TEST_P(EchoParamFixture, EchoNative) {
-  const auto params = GetParam();
-  auto multiplier = std::get<4>(params);
-
   proteus::NativeClient client;
   client.workerLoad("echo", nullptr);
 
@@ -135,21 +117,13 @@ TEST_P(EchoParamFixture, EchoNative) {
 
   auto response = client.modelInfer("echo", request);
 
-  if (multiplier == 30) {
-    EXPECT_TRUE(response.isError());
-    EXPECT_EQ(response.getError(), "Too many input tensors for this model");
-  } else {
-    validate(response);
-  }
+  validate(response);
 
   client.modelUnload("echo");
 }
 
 #ifdef PROTEUS_ENABLE_GRPC
 TEST_P(EchoParamFixture, EchoGrpc) {
-  const auto params = GetParam();
-  auto multiplier = std::get<4>(params);
-
   server_.startGrpc(50051);
   auto client = proteus::GrpcClient("localhost:50051");
   while (!client.serverLive()) {
@@ -160,17 +134,8 @@ TEST_P(EchoParamFixture, EchoGrpc) {
 
   auto request = this->construct_request();
 
-  if (multiplier == 30) {
-    try {
-      auto response = client.modelInfer("echo", request);
-      FAIL() << "No runtime error thrown";
-    } catch (const runtime_error& e) {
-      EXPECT_STREQ(e.what(), "Too many input tensors for this model");
-    }
-  } else {
-    auto response = client.modelInfer("echo", request);
-    validate(response);
-  }
+  auto response = client.modelInfer("echo", request);
+  validate(response);
 
   client.modelUnload("echo");
 }
@@ -178,10 +143,9 @@ TEST_P(EchoParamFixture, EchoGrpc) {
 
 // add_id, add_input_parameters, add_request_parameters, add_outputs, multiplier
 const std::tuple<bool, bool, bool, bool, int> configs[] = {
-  {true, true, true, true, 1},      {true, true, false, true, 1},
-  {true, false, false, true, 1},    {false, false, false, true, 1},
-  {false, false, false, false, 1},  {false, false, false, false, 10},
-  {false, false, false, false, 30},
+  {true, true, true, true, 1},     {true, true, false, true, 1},
+  {true, false, false, true, 1},   {false, false, false, true, 1},
+  {false, false, false, false, 1},
 };
 INSTANTIATE_TEST_SUITE_P(Echo, EchoParamFixture, testing::ValuesIn(configs));
 
diff --git a/tests/workers/test_echo.py b/tests/workers/test_echo.py
index 32740d987..2da6e6fe3 100644
--- a/tests/workers/test_echo.py
+++ b/tests/workers/test_echo.py
@@ -12,17 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pytest
 import numpy as np
-
-import proteus
+import pytest
+from helper import run_benchmark
 from proteus.predict_api import (
     InferenceRequest,
     InferenceRequestInput,
     InferenceRequestOutput,
 )
 
-from helper import run_benchmark
+import proteus
 
 
 @pytest.fixture(scope="class")
@@ -41,8 +40,8 @@ class TestEcho:
     Test the Echo worker
     """
 
-    inputs = [3, 5]
-    golden_outputs = [4, 6]
+    inputs = [3]
+    golden_outputs = [4]
 
     @classmethod
     def construct_request(
@@ -70,20 +69,9 @@ def construct_request(
             parameters.put("key", "value")
             input_0.parameters = parameters
 
-        input_1 = InferenceRequestInput()
-        input_1.name = "echo"
-        input_1.datatype = proteus.DataType.UINT32
-        input_1.shape = [1]
-        input_1.setUint32Data(np.array([cls.inputs[1]], np.uint32))
-        if add_input_parameters:
-            parameters = proteus.RequestParameters()
-            parameters.put("key2", 1)
-            input_1.parameters = parameters
-
         request = InferenceRequest()
         for _ in range(multiplier):
             request.addInputTensor(input_0)
-            request.addInputTensor(input_1)
 
         if add_id:
             request.id = "hello_world"
@@ -142,18 +130,17 @@ def send_request(self, request, id_present, input_tensors=1):
         assert response.model == "echo"
 
         outputs = response.getOutputs()
-        assert len(outputs) == 2 * input_tensors
+        assert len(outputs) == 1 * input_tensors
 
         for i in range(input_tensors):
-            for j in range(2):
-                output = outputs[(i * 2) + j]
-                data = output.getUint32Data()
-                assert len(data) == 1
-                assert data[0] == self.golden_outputs[j]
-                assert output.datatype == proteus.DataType.UINT32
-                assert output.name == "echo"
-                assert output.parameters.empty()
-                assert output.shape == [1]
+            output = outputs[i]
+            data = output.getUint32Data()
+            assert len(data) == 1
+            assert data[0] == self.golden_outputs[0]
+            assert output.datatype == proteus.DataType.UINT32
+            assert output.name == "echo"
+            assert output.parameters.empty()
+            assert output.shape == [1]
 
         if id_present:
             assert response.id == "hello_world"
@@ -213,33 +200,6 @@ def test_echo_4(self):
 
         self.send_request(request, add_id)
 
-    def test_echo_5(self):
-        """
-        Send a request to echo with no optional parameters with more input tensors
-        than echo initially allocates so it's forced to allocate more.
-        """
-        add_id = False
-        multiplier = 10
-        request = self.construct_request(add_id, False, False, multiplier)
-
-        self.send_request(request, add_id, multiplier)
-
-    def test_echo_6(self):
-        """
-        Send a request to echo with no optional parameters with too many input
-        tensors.
-        """
-        add_id = False
-        multiplier = 30
-        request = self.construct_request(add_id, False, False, multiplier)
-
-        try:
-            response = self.rest_client.modelInfer(self.model, request)
-        except proteus.RuntimeError as e:
-            assert str(e) == '{"error":"Too many input tensors for this model"}'
-        else:
-            pytest.fail("Exception not raised")
-
     @pytest.mark.benchmark(group="echo")
     def test_benchmark_echo_0(self, benchmark, model_fixture, parameters_fixture):
         request = self.construct_request(False, False, False)
diff --git a/tests/workers/test_facedetect.cpp b/tests/workers/test_facedetect.cpp
index 87130418d..ff73ba04a 100644
--- a/tests/workers/test_facedetect.cpp
+++ b/tests/workers/test_facedetect.cpp
@@ -12,8 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <cmath>    // for abs
-#include <cstdlib>  // for getenv
+#include <cstdlib>  // for getenv, abs
 
 #include "facedetect.hpp"  // IWYU pragma: associated
 #include "gtest/gtest.h"   // for Test, AssertionResult, EXPECT_EQ
diff --git a/tests/workers/test_facedetect.py b/tests/workers/test_facedetect.py
index b9ecf5340..f9c214ce3 100644
--- a/tests/workers/test_facedetect.py
+++ b/tests/workers/test_facedetect.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pytest
 import numpy as np
+import pytest
+from helper import root_path, run_benchmark
 
-from helper import run_benchmark, root_path
 import proteus
 
 
@@ -92,8 +92,7 @@ def send_request(self, request, check_asserts=True):
 
     def construct_request(self, asTensor):
         image_path = str(root_path / "tests/assets/girl-1867092_640.jpg")
-        # TODO(vishalk): AKS gives a segfault if batch != 4
-        batch = 4
+        batch = 1
         images = [image_path] * batch
         request = proteus.ImageInferenceRequest(images, asTensor)
 
diff --git a/tests/workers/test_facedetect_stream.py b/tests/workers/test_facedetect_stream.py
index 523c0c2a0..f8f9ff9fd 100644
--- a/tests/workers/test_facedetect_stream.py
+++ b/tests/workers/test_facedetect_stream.py
@@ -15,10 +15,10 @@
 import json
 
 import pytest
-
 from helper import root_path
+from proteus.predict_api import InferenceRequest, InferenceRequestInput
+
 import proteus
-from proteus.predict_api import InferenceRequestInput, InferenceRequest
 
 
 @pytest.fixture(scope="class")
@@ -60,7 +60,7 @@ def construct_request(self, requested_frames_count):
         parameters_2.put("key", "0")
         request.parameters = parameters_2
 
-        self.ws_client.modelInferAsync(self.model, request)
+        self.ws_client.modelInferWs(self.model, request)
         response_str = self.ws_client.modelRecv()
         response = json.loads(response_str)
 
diff --git a/tests/workers/test_fake.py b/tests/workers/test_fake.py
deleted file mode 100644
index 2249bd37e..000000000
--- a/tests/workers/test_fake.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Copyright 2021 Xilinx Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import pytest
-from proteus.predict_api import (
-    InferenceRequest,
-    InferenceRequestInput,
-    InferenceRequestOutput,
-)
-
-from helper import run_benchmark
-
-
-@pytest.fixture(scope="class")
-def model_fixture():
-    return "fake"
-
-
-@pytest.fixture(scope="class")
-def parameters_fixture():
-    return None
-
-
-# @pytest.mark.usefixtures("load")
-# class TestFake:
-#     """
-#     Test the Fake worker
-#     """
-
-#     def test_fake_0(self):
-#         """
-#         Send a request to echo with no optional parameters with too many input
-#         tensors.
-#         """
-#         input_0 = RequestInput("fake")
-#         input_0.datatype = Datatype.UINT32
-#         input_0.shape.append(1)
-#         input_0.data.append(1)
-#         request = InferenceRequest(input_0)
-#         response = self.rest_client._infer("fake", request)
-#         assert response.status_code == 200
diff --git a/tests/workers/test_invert_video.py b/tests/workers/test_invert_video.py
index fa2359fd0..4c11c31e5 100644
--- a/tests/workers/test_invert_video.py
+++ b/tests/workers/test_invert_video.py
@@ -16,11 +16,11 @@
 
 import cv2
 import pytest
-
 from helper import root_path
+from proteus.predict_api import InferenceRequest, InferenceRequestInput
 from test_invert_image import compare_jpgs
+
 import proteus
-from proteus.predict_api import InferenceRequestInput, InferenceRequest
 
 
 @pytest.fixture(scope="class")
@@ -56,7 +56,7 @@ def construct_request(self, video_path, requested_frames_count):
         parameters_2.put("key", "0")
         request.parameters = parameters_2
 
-        self.ws_client.modelInferAsync(self.model, request)
+        self.ws_client.modelInferWs(self.model, request)
         response_str = self.ws_client.modelRecv()
         response = json.loads(response_str)
 
diff --git a/tests/workers/test_migraphx.py b/tests/workers/test_migraphx.py
index 3862fede4..59b16a0b6 100644
--- a/tests/workers/test_migraphx.py
+++ b/tests/workers/test_migraphx.py
@@ -131,15 +131,13 @@ def test_migraphx(self, num):
         """
         image_paths = [
             str(root_path / "tests/assets/dog-3619020_640.jpg"),
-            str(root_path / "tests/assets/bicycle-384566_640.jpg"),
         ]
-        gold_responses = [[259, 261, 157, 260, 154], [671, 444, 880, 518, 870]]
+        gold_responses = [[259, 261, 157, 260, 154]]
         assert len(image_paths) == len(gold_responses)
         image_num = len(image_paths)
 
-        batch = num
-        images = []
-        for i in range(batch):
+        request_num = num
+        for i in range(request_num):
             # Load a picture
             img = cv2.imread(image_paths[i % image_num]).astype("float32")
             img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
@@ -148,19 +146,19 @@ def test_migraphx(self, num):
             #  Normalize contents with values specific to Resnet50
             img = img.transpose(2, 0, 1)
             img = preprocess(img)
-            images.append(img)
-        request = proteus.ImageInferenceRequest(images, True)
-        response = self.send_request(request)
-        top_k_responses = postprocess(response, 5)
 
-        for i, top_k in enumerate(top_k_responses):
-            # Look for the top 5 categories (not underlying scores)
-            assert (top_k == gold_responses[i % image_num]).all()
+            request = proteus.ImageInferenceRequest(img, True)
+            response = self.send_request(request)
+            top_k_responses = postprocess(response, 5)
+
+            for i, top_k in enumerate(top_k_responses):
+                # Look for the top 5 categories (not underlying scores)
+                assert (top_k == gold_responses[i % image_num]).all()
 
     @pytest.mark.benchmark(group="Migraphx")
     def test_benchmark_Migraphx(self, benchmark, model_fixture, parameters_fixture):
 
-        batch_size = 16
+        batch_size = 1
         input_size = parameters_fixture.get("input_size")
         images = np.random.uniform(
             0.0, 255.0, (batch_size, input_size, input_size, 3)
diff --git a/tests/workers/test_ptzendnn.py b/tests/workers/test_ptzendnn.py
index a96a195ff..85cada6a5 100644
--- a/tests/workers/test_ptzendnn.py
+++ b/tests/workers/test_ptzendnn.py
@@ -81,7 +81,7 @@ def send_request(self, request, check_asserts=True):
                 assert output.parameters.empty()
         return response
 
-    @pytest.mark.parametrize("num", [1, 8])
+    @pytest.mark.parametrize("num", [1])
     def test_ptzendnn_0(self, num):
         """
         Send a request to pt model as tensor data
diff --git a/tests/workers/test_resnet50_dpucadf8h.py b/tests/workers/test_resnet50_dpucadf8h.py
index 0c52bcfc2..aa4f28e3f 100644
--- a/tests/workers/test_resnet50_dpucadf8h.py
+++ b/tests/workers/test_resnet50_dpucadf8h.py
@@ -79,7 +79,7 @@ def send_request(self, request, check_asserts=True):
                 np.testing.assert_almost_equal(gold_response_output, data, 2)
         return response
 
-    def construct_request(self, asTensor, batches=4):
+    def construct_request(self, asTensor, batches=1):
         image_path = str(root_path / "tests/assets/dog-3619020_640.jpg")
 
         # TODO(vishalk): AKS gives a segfault if batch != 4
diff --git a/tests/workers/test_resnet50_stream.py b/tests/workers/test_resnet50_stream.py
index 7289eb06a..e22c7d8db 100644
--- a/tests/workers/test_resnet50_stream.py
+++ b/tests/workers/test_resnet50_stream.py
@@ -57,7 +57,7 @@ def construct_request(self, requested_frames_count):
         parameters_2.put("key", "0")
         request.parameters = parameters_2
 
-        self.ws_client.modelInferAsync(self.model, request)
+        self.ws_client.modelInferWs(self.model, request)
         response_str = self.ws_client.modelRecv()
         response = json.loads(response_str)
 
diff --git a/tests/workers/test_tfzendnn.py b/tests/workers/test_tfzendnn.py
index 2ffb860c0..99bb8a0f0 100644
--- a/tests/workers/test_tfzendnn.py
+++ b/tests/workers/test_tfzendnn.py
@@ -87,7 +87,7 @@ def send_request(self, request, check_asserts=True):
                 assert output.parameters.empty()
         return response
 
-    @pytest.mark.parametrize("num", [1, 8])
+    @pytest.mark.parametrize("num", [1])
     def test_tfzendnn_0(self, num):
         """
         Send a request to tf model as tensor data
diff --git a/tests/workers/test_xmodel.py b/tests/workers/test_xmodel.py
index 2e490949f..a97d4b5a7 100644
--- a/tests/workers/test_xmodel.py
+++ b/tests/workers/test_xmodel.py
@@ -151,21 +151,22 @@ def test_xmodel_0(self):
 
         batch = 4
         images = []
-        for _ in range(batch):
+        for _ in range(1):
             image = cv2.imread(image_path)
             images.append(image)
 
         images = self.preprocess(images, preprocessing)
-        request = proteus.ImageInferenceRequest(images, True)
-        response = self.send_request(request)
-        outputs = response.getOutputs()
-        assert len(outputs) == batch
-        output = outputs[0]
-        assert output.datatype == proteus.DataType.INT8
-        data = output.getInt8Data()
-        k = self.postprocess(data, 5)
-        gold_response_output = [259, 261, 260, 157, 154]
-        assert k == gold_response_output
+        for _ in range(batch):
+            request = proteus.ImageInferenceRequest(images[0], True)
+            response = self.send_request(request)
+            outputs = response.getOutputs()
+            assert len(outputs) == 1
+            output = outputs[0]
+            assert output.datatype == proteus.DataType.INT8
+            data = output.getInt8Data()
+            k = self.postprocess(data, 5)
+            gold_response_output = [259, 261, 260, 157, 154]
+            assert k == gold_response_output
 
     @pytest.mark.benchmark(group="xmodel")
     def test_benchmark_xmodel(self, benchmark, model_fixture, parameters_fixture):
diff --git a/tests/workers/test_yolov3.py b/tests/workers/test_yolov3.py
index b85d20bba..e6a848cb6 100644
--- a/tests/workers/test_yolov3.py
+++ b/tests/workers/test_yolov3.py
@@ -101,8 +101,7 @@ def send_request(self, request, check_asserts=True):
     def construct_request(self, asTensor):
         image_path = str(root_path / "tests/assets/bicycle-384566_640.jpg")
 
-        # TODO(vishalk): AKS gives a segfault if batch != 4
-        batch = 4
+        batch = 1
         images = [image_path] * batch
         request = proteus.ImageInferenceRequest(images, asTensor)
 
diff --git a/tests/workers/xmodel.hpp b/tests/workers/xmodel.hpp
index 32fdfa648..3dcea2d95 100644
--- a/tests/workers/xmodel.hpp
+++ b/tests/workers/xmodel.hpp
@@ -151,8 +151,9 @@ using FutureQueue =
 
 void enqueue(int images, const std::string& workerName,
              proteus::InferenceRequest request, FutureQueue& my_queue) {
+  proteus::NativeClient client;
   for (int i = 0; i < images; i++) {
-    auto future = proteus::NativeClient::enqueue(workerName, request);
+    auto future = client.modelInferAsync(workerName, request);
     my_queue.enqueue(std::move(future));
   }
 }