Repository: Xilinx/inference-server (language: python)

Issue:
Use memory manager for workers
Currently, workers are responsible for allocating their own memory that they contribute to a pool for their worker group. There are a few issues around this:

1. Doesn't work well for non-constant input sizes
2. Potentially a lot of extra memory is kept around
3. Can't easily use hardware-pinned memory

One solution is a central memory manager.

Goals:
1. Allow for dynamic batching (#131) by allowing the batcher to return unused memory
2. Allow for hardware backed memory but also CPU based memory
3. Allow for chaining 
4. Allocate more memory if needed


Discussion / hints:

Fix patch:
diff --git a/docs/dependencies.rst b/docs/dependencies.rst
index 2cf93c334..b694121a7 100644
--- a/docs/dependencies.rst
+++ b/docs/dependencies.rst
@@ -212,8 +212,8 @@ The following packages are downloaded from AMD.
     :header: Name,Version,License,Usage
     :widths: auto
 
-    `PT_v1.11.0_ZenDNN_v3.3_C++_API.zip <https://www.amd.com/en/developer/zendnn.html>`__,4.0,AMD ZenDNN EULA,Used by PT+ZenDNN worker\ :superscript:`a 3`
-    `TF_v2.9_ZenDNN_v3.3_C++_API.zip <https://www.amd.com/en/developer/zendnn.html>`__,4.0,AMD ZenDNN EULA,Used by TF+ZenDNN worker\ :superscript:`a 2`
+    `PT_v1.12.0_ZenDNN_v4.0_C++_API.zip <https://www.amd.com/en/developer/zendnn.html>`__,4.0,AMD ZenDNN EULA,Used by PT+ZenDNN worker\ :superscript:`a 3`
+    `TF_v2.10_ZenDNN_v4.0_C++_API.zip <https://www.amd.com/en/developer/zendnn.html>`__,4.0,AMD ZenDNN EULA,Used by TF+ZenDNN worker\ :superscript:`a 2`
 
 
 Included
diff --git a/examples/resnet50/ptzendnn.py b/examples/resnet50/ptzendnn.py
index 3b4757709..d397014ff 100644
--- a/examples/resnet50/ptzendnn.py
+++ b/examples/resnet50/ptzendnn.py
@@ -153,7 +153,7 @@ def main(args):
     server = amdinfer.Server()
     print("Waiting until the server is ready...")
 
-    client = amdinfer.NativeClient()
+    client = amdinfer.NativeClient(server)
     amdinfer.waitUntilServerReady(client)
 
     if args.endpoint:
diff --git a/include/amdinfer/declarations.hpp b/include/amdinfer/declarations.hpp
index 3f8a152a9..65620f7dd 100644
--- a/include/amdinfer/declarations.hpp
+++ b/include/amdinfer/declarations.hpp
@@ -49,6 +49,9 @@ using InferenceResponseFuture = std::future<amdinfer::InferenceResponse>;
 
 using StringMap = std::unordered_map<std::string, std::string>;
 
+class Trace;
+using TracePtr = std::unique_ptr<Trace>;
+
 using Kernels = std::unordered_map<std::string, int>;
 
 }  // namespace amdinfer
diff --git a/src/amdinfer/CMakeLists.txt b/src/amdinfer/CMakeLists.txt
index 0048203df..a49a48540 100644
--- a/src/amdinfer/CMakeLists.txt
+++ b/src/amdinfer/CMakeLists.txt
@@ -50,6 +50,7 @@ add_library(
   amdinfer
   ${TYPE}
   ${BATCHING_OBJS}
+  ${BUFFERS_OBJS}
   ${CLIENT_OBJS}
   ${CORE_OBJS}
   ${HELPER_OBJS}
@@ -66,6 +67,7 @@ add_library(
   amdinfer-server
   ${TYPE}
   ${BATCHING_OBJS}
+  ${BUFFERS_OBJS}
   ${CORE_OBJS}
   ${HELPER_OBJS}
   ${OBSERVATION_OBJS}
diff --git a/src/amdinfer/batching/CMakeLists.txt b/src/amdinfer/batching/CMakeLists.txt
index 67fa62e17..7abe885f4 100644
--- a/src/amdinfer/batching/CMakeLists.txt
+++ b/src/amdinfer/batching/CMakeLists.txt
@@ -16,10 +16,15 @@
 list(APPEND files hard soft)
 list(APPEND targets)
 
+add_library(batch OBJECT batch.cpp)
+target_include_directories(batch PRIVATE ${AMDINFER_INCLUDE_DIRS})
+set_target_options(batch)
+
 add_library(batcher OBJECT batcher.cpp)
 target_include_directories(batcher PRIVATE ${AMDINFER_INCLUDE_DIRS})
 set_target_options(batcher)
-set(BATCHING_OBJS $<TARGET_OBJECTS:batcher>)
+
+set(BATCHING_OBJS $<TARGET_OBJECTS:batch> $<TARGET_OBJECTS:batcher>)
 
 foreach(file ${files})
   set(target ${file}Batcher)
@@ -34,7 +39,7 @@ endforeach()
 target_link_libraries(softBatcher INTERFACE timer)
 
 add_library(batching STATIC ${BATCHING_OBJS})
-target_link_libraries(batching INTERFACE batcher ${targets})
+target_link_libraries(batching INTERFACE batch batcher ${targets})
 set_target_options(batching)
 
 set(BATCHING_OBJS ${BATCHING_OBJS} PARENT_SCOPE)
diff --git a/src/amdinfer/batching/batch.cpp b/src/amdinfer/batching/batch.cpp
new file mode 100644
index 000000000..1f62d2f73
--- /dev/null
+++ b/src/amdinfer/batching/batch.cpp
@@ -0,0 +1,99 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#include "amdinfer/batching/batch.hpp"
+
+#include "amdinfer/buffers/buffer.hpp"
+#include "amdinfer/observation/tracing.hpp"
+
+namespace amdinfer {
+
+void Batch::addRequest(InferenceRequestPtr request) {
+  requests_.push_back(std::move(request));
+}
+
+std::vector<Buffer*> Batch::getRawInputBuffers() const {
+  std::vector<Buffer*> buffers;
+  buffers.reserve(input_buffers_.size());
+  for (const auto& buffer : input_buffers_) {
+    buffers.push_back(buffer.get());
+  }
+  return buffers;
+}
+
+BufferPtrs Batch::getInputBuffers() { return std::move(input_buffers_); }
+
+std::vector<Buffer*> Batch::getRawOutputBuffers() const {
+  std::vector<Buffer*> buffers;
+  buffers.reserve(output_buffers_.size());
+  for (const auto& buffer : output_buffers_) {
+    buffers.push_back(buffer.get());
+  }
+  return buffers;
+}
+
+BufferPtrs Batch::getOutputBuffers() { return std::move(output_buffers_); }
+
+const std::vector<InferenceRequestPtr>& Batch::getRequests() const {
+  return requests_;
+}
+
+const InferenceRequestPtr& Batch::getRequest(size_t index) {
+  return requests_.at(index);
+}
+
+bool Batch::empty() const { return requests_.empty(); }
+
+size_t Batch::size() const {
+#ifdef AMDINFER_ENABLE_TRACING
+  assert(requests_.size() == traces_.size());
+#endif
+#ifdef AMDINFER_ENABLE_METRICS
+  assert(requests_.size() == start_times_.size());
+#endif
+
+  return requests_.size();
+}
+
+size_t Batch::getInputSize() const { return input_buffers_.size(); }
+
+size_t Batch::getOutputSize() const { return output_buffers_.size(); }
+
+void Batch::setBuffers(BufferPtrs inputs, BufferPtrs outputs) {
+  input_buffers_ = std::move(inputs);
+  output_buffers_ = std::move(outputs);
+}
+
+#ifdef AMDINFER_ENABLE_TRACING
+void Batch::addTrace(TracePtr trace) { traces_.push_back(std::move(trace)); }
+
+TracePtr& Batch::getTrace(size_t index) { return traces_.at(index); }
+#endif
+
+#ifdef AMDINFER_ENABLE_METRICS
+void Batch::addTime(std::chrono::high_resolution_clock::time_point timestamp) {
+  start_times_.push_back(timestamp);
+}
+
+std::chrono::high_resolution_clock::time_point Batch::getTime(size_t index) {
+  return start_times_.at(index);
+}
+#endif
+
+}  // namespace amdinfer
diff --git a/src/amdinfer/batching/batch.hpp b/src/amdinfer/batching/batch.hpp
new file mode 100644
index 000000000..af307ff37
--- /dev/null
+++ b/src/amdinfer/batching/batch.hpp
@@ -0,0 +1,80 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#ifndef GUARD_AMDINFER_BATCHING_BATCH
+#define GUARD_AMDINFER_BATCHING_BATCH
+
+#include "amdinfer/build_options.hpp"
+#include "amdinfer/declarations.hpp"
+
+namespace amdinfer {
+
+/**
+ * @brief The Batch is what the batcher produces and pushes to the workers. It
+ * represents the requests, the buffers associated with the request and other
+ * metadata that should be sent to the worker.
+ *
+ */
+class Batch {
+ public:
+  void addRequest(InferenceRequestPtr request);
+
+  void setBuffers(BufferPtrs inputs, BufferPtrs outputs);
+  [[nodiscard]] const InferenceRequestPtr& getRequest(size_t index);
+  [[nodiscard]] const std::vector<InferenceRequestPtr>& getRequests() const;
+  [[nodiscard]] std::vector<BufferPtr> getInputBuffers();
+  [[nodiscard]] std::vector<BufferPtr> getOutputBuffers();
+  [[nodiscard]] std::vector<Buffer*> getRawInputBuffers() const;
+  [[nodiscard]] std::vector<Buffer*> getRawOutputBuffers() const;
+
+  [[nodiscard]] bool empty() const;
+  [[nodiscard]] size_t size() const;
+  [[nodiscard]] size_t getInputSize() const;
+  [[nodiscard]] size_t getOutputSize() const;
+
+#ifdef AMDINFER_ENABLE_TRACING
+  void addTrace(TracePtr trace);
+  TracePtr& getTrace(size_t index);
+#endif
+#ifdef AMDINFER_ENABLE_METRICS
+  void addTime(std::chrono::high_resolution_clock::time_point timestamp);
+  std::chrono::high_resolution_clock::time_point getTime(size_t index);
+#endif
+
+  [[nodiscard]] auto begin() const { return requests_.begin(); }
+  [[nodiscard]] auto end() const { return requests_.end(); }
+
+ private:
+  const WorkerInfo* worker_;
+  std::vector<InferenceRequestPtr> requests_;
+  std::vector<BufferPtr> input_buffers_;
+  std::vector<BufferPtr> output_buffers_;
+#ifdef AMDINFER_ENABLE_TRACING
+  std::vector<TracePtr> traces_;
+#endif
+#ifdef AMDINFER_ENABLE_METRICS
+  std::vector<std::chrono::high_resolution_clock::time_point> start_times_;
+#endif
+};
+
+using BatchPtr = std::unique_ptr<Batch>;
+
+}  // namespace amdinfer
+
+#endif  // GUARD_AMDINFER_BATCHING_BATCH
diff --git a/src/amdinfer/batching/batcher.cpp b/src/amdinfer/batching/batcher.cpp
index 334842a3f..829927d3b 100644
--- a/src/amdinfer/batching/batcher.cpp
+++ b/src/amdinfer/batching/batcher.cpp
@@ -25,10 +25,11 @@
 #include <string>   // for string
 #include <utility>  // for move
 
-#include "amdinfer/buffers/buffer.hpp"       // IWYU pragma: keep
-#include "amdinfer/core/interface.hpp"       // IWYU pragma: keep
-#include "amdinfer/core/worker_info.hpp"     // for WorkerInfo
-#include "amdinfer/observation/logging.hpp"  // for Logger, Loggers, Loggers:...
+#include "amdinfer/buffers/buffer.hpp"         // IWYU pragma: keep
+#include "amdinfer/core/interface.hpp"         // IWYU pragma: keep
+#include "amdinfer/core/memory_pool/pool.hpp"  // for MemoryPool
+#include "amdinfer/core/worker_info.hpp"       // for WorkerInfo
+#include "amdinfer/observation/logging.hpp"    // for Logger, Loggers, Logger...
 
 namespace amdinfer {
 
@@ -38,7 +39,7 @@ namespace amdinfer {
  *
  */
 
-Batcher::Batcher() {
+Batcher::Batcher(MemoryPool* pool) : pool_(pool) {
   this->input_queue_ = std::make_shared<BlockingQueue<InterfacePtr>>();
   this->output_queue_ = std::make_shared<BatchPtrQueue>();
   this->status_ = BatcherStatus::New;
@@ -47,26 +48,27 @@ Batcher::Batcher() {
 #endif
 }
 
-Batcher::Batcher(ParameterMap* parameters) : Batcher() {
+Batcher::Batcher(MemoryPool* pool, ParameterMap* parameters) : Batcher(pool) {
   if (parameters != nullptr) {
     this->parameters_ = *parameters;
   }
 }
 
-Batcher::Batcher(const Batcher& batcher) {
-  this->input_queue_ = batcher.input_queue_;
-  this->output_queue_ = batcher.output_queue_;
-  this->batch_size_ = batcher.batch_size_;
+Batcher::Batcher(const Batcher& batcher)
+  : batch_size_(batcher.batch_size_),
+    input_queue_(batcher.input_queue_),
+    output_queue_(batcher.output_queue_),
+    model_(batcher.model_),
+    pool_(batcher.pool_) {
   this->status_ = BatcherStatus::New;
 #ifdef AMDINFER_ENABLE_LOGGING
   this->logger_ = Logger(Loggers::Server);
 #endif
-  this->model_ = batcher.model_;
 }
 
-void Batcher::start(WorkerInfo* worker) {
+void Batcher::start(const std::vector<MemoryAllocators>& allocators) {
   this->status_ = BatcherStatus::Run;
-  this->thread_ = std::thread(&Batcher::run, this, worker);
+  this->thread_ = std::thread(&Batcher::run, this, allocators);
 }
 
 void Batcher::setBatchSize(size_t batch_size) {
@@ -83,16 +85,16 @@ BlockingQueue<InterfacePtr>* Batcher::getInputQueue() {
 
 BatchPtrQueue* Batcher::getOutputQueue() { return this->output_queue_.get(); }
 
-void Batcher::enqueue(InterfacePtr request) {
+void Batcher::enqueue(InterfacePtr request) const {
   this->input_queue_->enqueue(std::move(request));
 }
 
-void Batcher::run(WorkerInfo* worker) {
-  this->doRun(worker);
+void Batcher::run(const std::vector<MemoryAllocators>& allocators) {
+  this->doRun(allocators);
   this->status_ = BatcherStatus::Inactive;
 }
 
-BatcherStatus Batcher::getStatus() { return this->status_; }
+BatcherStatus Batcher::getStatus() const { return this->status_; }
 
 void Batcher::end() {
   this->thread_.join();
@@ -103,81 +105,4 @@ void Batcher::end() {
 const Logger& Batcher::getLogger() const { return logger_; }
 #endif
 
-Batch::Batch(const WorkerInfo* worker)
-  : worker_(worker),
-    input_buffers_(worker->getInputBuffer()),
-    output_buffers_(worker->getOutputBuffer()) {}
-
-Batch::~Batch() {
-  worker_->putInputBuffer(std::move(input_buffers_));
-  worker_->putOutputBuffer(std::move(output_buffers_));
-}
-
-void Batch::addRequest(InferenceRequestPtr request) {
-  requests_.push_back(std::move(request));
-}
-
-std::vector<Buffer*> Batch::getRawInputBuffers() const {
-  std::vector<Buffer*> buffers;
-  buffers.reserve(input_buffers_.size());
-  for (const auto& buffer : input_buffers_) {
-    buffers.push_back(buffer.get());
-  }
-  return buffers;
-}
-
-const BufferPtrs& Batch::getInputBuffers() const { return input_buffers_; }
-
-std::vector<Buffer*> Batch::getRawOutputBuffers() const {
-  std::vector<Buffer*> buffers;
-  buffers.reserve(output_buffers_.size());
-  for (const auto& buffer : output_buffers_) {
-    buffers.push_back(buffer.get());
-  }
-  return buffers;
-}
-
-const BufferPtrs& Batch::getOutputBuffers() const { return output_buffers_; }
-
-const std::vector<InferenceRequestPtr>& Batch::getRequests() const {
-  return requests_;
-}
-
-const InferenceRequestPtr& Batch::getRequest(size_t index) {
-  return requests_.at(index);
-}
-
-bool Batch::empty() const { return requests_.empty(); }
-
-size_t Batch::size() const {
-#ifdef AMDINFER_ENABLE_TRACING
-  assert(requests_.size() == traces_.size());
-#endif
-#ifdef AMDINFER_ENABLE_METRICS
-  assert(requests_.size() == start_times_.size());
-#endif
-
-  return requests_.size();
-}
-
-size_t Batch::getInputSize() const { return input_buffers_.size(); }
-
-size_t Batch::getOutputSize() const { return output_buffers_.size(); }
-
-#ifdef AMDINFER_ENABLE_TRACING
-void Batch::addTrace(TracePtr trace) { traces_.push_back(std::move(trace)); }
-
-TracePtr& Batch::getTrace(size_t index) { return traces_.at(index); }
-#endif
-
-#ifdef AMDINFER_ENABLE_METRICS
-void Batch::addTime(std::chrono::high_resolution_clock::time_point timestamp) {
-  start_times_.push_back(timestamp);
-}
-
-std::chrono::high_resolution_clock::time_point Batch::getTime(size_t index) {
-  return start_times_.at(index);
-}
-#endif
-
 }  // namespace amdinfer
diff --git a/src/amdinfer/batching/batcher.hpp b/src/amdinfer/batching/batcher.hpp
index 08f5f083a..21faca3ce 100644
--- a/src/amdinfer/batching/batcher.hpp
+++ b/src/amdinfer/batching/batcher.hpp
@@ -21,13 +21,13 @@
 #ifndef GUARD_AMDINFER_BATCHING_BATCHER
 #define GUARD_AMDINFER_BATCHING_BATCHER
 
-#include <chrono>   // for system_clock::time_point
 #include <cstddef>  // for size_t
 #include <memory>   // for unique_ptr, shared_ptr
 #include <string>   // for string
 #include <thread>   // for thread
 #include <vector>   // for vector
 
+#include "amdinfer/batching/batch.hpp"       // for Batch
 #include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_LOGGING
 #include "amdinfer/core/parameters.hpp"      // for ParameterMap
 #include "amdinfer/declarations.hpp"         // for BufferPtrs, InferenceReq...
@@ -38,72 +38,14 @@
 namespace amdinfer {
 class Buffer;
 class WorkerInfo;
+class MemoryPool;
+enum class MemoryAllocators;
 }  // namespace amdinfer
 
 namespace amdinfer {
 
 enum class BatcherStatus { New, Run, Inactive, Dead };
 
-/**
- * @brief The Batch is what the batcher produces and pushes to the workers. It
- * represents the requests, the buffers associated with the request and other
- * metadata that should be sent to the worker.
- *
- */
-class Batch {
- public:
-  explicit Batch(const WorkerInfo* worker);
-  /// Copy constructor
-  Batch(Batch const&) = delete;
-  /// Copy assignment constructor
-  Batch& operator=(const Batch&) = delete;
-  /// Move constructor
-  Batch(Batch&& other) = default;
-  /// Move assignment constructor
-  Batch& operator=(Batch&& other) = default;
-  /// Destructor
-  ~Batch();
-
-  void addRequest(InferenceRequestPtr request);
-
-  [[nodiscard]] const InferenceRequestPtr& getRequest(size_t index);
-  [[nodiscard]] const std::vector<InferenceRequestPtr>& getRequests() const;
-  [[nodiscard]] const BufferPtrs& getInputBuffers() const;
-  [[nodiscard]] const BufferPtrs& getOutputBuffers() const;
-  [[nodiscard]] std::vector<Buffer*> getRawInputBuffers() const;
-  [[nodiscard]] std::vector<Buffer*> getRawOutputBuffers() const;
-
-  [[nodiscard]] bool empty() const;
-  [[nodiscard]] size_t size() const;
-  [[nodiscard]] size_t getInputSize() const;
-  [[nodiscard]] size_t getOutputSize() const;
-
-#ifdef AMDINFER_ENABLE_TRACING
-  void addTrace(TracePtr trace);
-  TracePtr& getTrace(size_t index);
-#endif
-#ifdef AMDINFER_ENABLE_METRICS
-  void addTime(std::chrono::high_resolution_clock::time_point timestamp);
-  std::chrono::high_resolution_clock::time_point getTime(size_t index);
-#endif
-
-  [[nodiscard]] auto begin() const { return requests_.begin(); }
-  [[nodiscard]] auto end() const { return requests_.end(); }
-
- private:
-  const WorkerInfo* worker_;
-  std::vector<InferenceRequestPtr> requests_;
-  std::vector<BufferPtr> input_buffers_;
-  std::vector<BufferPtr> output_buffers_;
-#ifdef AMDINFER_ENABLE_TRACING
-  std::vector<TracePtr> traces_;
-#endif
-#ifdef AMDINFER_ENABLE_METRICS
-  std::vector<std::chrono::high_resolution_clock::time_point> start_times_;
-#endif
-};
-
-using BatchPtr = std::unique_ptr<Batch>;
 using BatchPtrQueue = BlockingQueue<BatchPtr>;
 
 /**
@@ -115,8 +57,8 @@ using BatchPtrQueue = BlockingQueue<BatchPtr>;
 class Batcher {
  public:
   /// Construct a new Batcher object
-  Batcher();
-  explicit Batcher(ParameterMap* parameters);
+  explicit Batcher(MemoryPool* pool);
+  Batcher(MemoryPool* pool, ParameterMap* parameters);
   /**
    * @brief Construct a new Batcher object
    *
@@ -135,7 +77,7 @@ class Batcher {
    *
    * @param worker
    */
-  void start(WorkerInfo* worker);
+  void start(const std::vector<MemoryAllocators>& allocator);
   /**
    * @brief Set the batch size for the batcher
    *
@@ -157,16 +99,16 @@ class Batcher {
   /// Get the batcher's output queue (used to push batches to the worker group)
   BatchPtrQueue* getOutputQueue();
 
-  void run(WorkerInfo* worker);
+  void run(const std::vector<MemoryAllocators>& allocators);
 
-  BatcherStatus getStatus();
+  BatcherStatus getStatus() const;
 
   /**
    * @brief Enqueue a new request to the batcher
    *
    * @param request
    */
-  void enqueue(InterfacePtr request);
+  void enqueue(InterfacePtr request) const;
 
   /// End the batcher
   void end();
@@ -182,15 +124,16 @@ class Batcher {
   std::thread thread_;
   std::string model_;
   ParameterMap parameters_;
+  MemoryPool* pool_;
 
  private:
   /**
    * @brief The doRun method defines the exact process by which the batcher
    * consumes incoming Interface objects and uses them to create batches.
    *
-   * @param worker pointer to this batcher's worker [group]
+   * @param allocators vector of allocators that may be used to get memory
    */
-  virtual void doRun(WorkerInfo* worker) = 0;
+  virtual void doRun(const std::vector<MemoryAllocators>& allocators) = 0;
 
   BatcherStatus status_;
 
diff --git a/src/amdinfer/batching/hard.cpp b/src/amdinfer/batching/hard.cpp
index ba023bff9..05a986ccc 100644
--- a/src/amdinfer/batching/hard.cpp
+++ b/src/amdinfer/batching/hard.cpp
@@ -26,33 +26,37 @@
 #include <utility>  // for move
 #include <vector>   // for vector
 
-#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_METRICS
-#include "amdinfer/core/exceptions.hpp"      // for invalid_argument
-#include "amdinfer/core/interface.hpp"       // for Interface
-#include "amdinfer/declarations.hpp"         // for InterfacePtr
-#include "amdinfer/observation/metrics.hpp"  // for Metrics, MetricCounterIDs
-#include "amdinfer/observation/tracing.hpp"  // for Trace
-#include "amdinfer/util/queue.hpp"           // for BlockingConcurrentQueue
-#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_METRICS
+#include "amdinfer/core/exceptions.hpp"        // for invalid_argument
+#include "amdinfer/core/interface.hpp"         // for Interface
+#include "amdinfer/core/memory_pool/pool.hpp"  // for MemoryPool
+#include "amdinfer/core/worker_info.hpp"       // for WorkerInfo
+#include "amdinfer/declarations.hpp"           // for InterfacePtr
+#include "amdinfer/observation/metrics.hpp"    // for Metrics, MetricCounterIDs
+#include "amdinfer/observation/tracing.hpp"    // for Trace
+#include "amdinfer/util/queue.hpp"             // for BlockingConcurrentQueue
+#include "amdinfer/util/thread.hpp"            // for setThreadName
 
 // IWYU pragma: no_forward_declare amdinfer::Buffer
 
 namespace amdinfer {
 
-void HardBatcher::doRun(WorkerInfo* worker) {
+void HardBatcher::doRun(const std::vector<MemoryAllocators>& allocators) {
   auto thread_name = "batch" + this->getName();
   util::setThreadName(thread_name);
   InterfacePtr req;
   bool run = true;
 
   while (run) {
-    auto batch = std::make_unique<Batch>(worker);
-    auto input_buffers = batch->getRawInputBuffers();
-    auto output_buffers = batch->getRawOutputBuffers();
-    std::vector<size_t> input_offset(input_buffers.size(), 0);
-    std::vector<size_t> output_offset(output_buffers.size(), 0);
+    auto batch = std::make_unique<Batch>();
     size_t batch_size = 0;
 
+    std::vector<BufferPtr> input_buffers;
+    std::vector<size_t> input_offset;
+    std::vector<size_t> output_offset;
+
+    bool first_request = true;
+
     do {
       this->input_queue_->wait_dequeue(req);
 
@@ -72,25 +76,44 @@ void HardBatcher::doRun(WorkerInfo* worker) {
 #endif
 
       auto input_size = req->getInputSize();
-      if (input_size != input_buffers.size()) {
-        auto e =
-          invalid_argument("Number of input tensors do not match worker");
-        req->errorHandler(e);
-        continue;
-      }
       if (input_size == 0) {
         auto error = invalid_argument("Input size is zero");
         req->errorHandler(error);
         continue;
       }
 
+      if (first_request) {
+        auto input_sizes = req->getInputSizes();
+        input_buffers.reserve(input_sizes.size());
+        // auto output_sizes = req->getOutputSizes();
+        // TODO(varunsh): the spec does not require the request to have outputs
+        // additionally, the output size could be variable so this should be
+        // allocated by the worker
+
+        // std::vector<BufferPtr> output_buffers;
+        // output_buffers.reserve(output_sizes.size());
+        // std::vector<size_t> output_offset(output_buffers.size(), 0);
+        for (const auto& tensor_size : input_sizes) {
+          input_buffers.push_back(
+            pool_->get(allocators, tensor_size * batch_size_));
+        }
+        // for(const auto& tensor_size : output_sizes) {
+        //   output_buffers.push_back(pool_->get(allocators, tensor_size));
+        // }
+        input_offset.resize(input_buffers.size());
+        batch->setBuffers(std::move(input_buffers), {});
+      }
+
+      auto raw_inputs = batch->getRawInputBuffers();
+      auto raw_outputs = batch->getRawOutputBuffers();
+
       auto old_input_offset = input_offset;
-      auto old_output_offset = output_offset;
-      auto new_req = req->getRequest(input_buffers, input_offset,
-                                     output_buffers, output_offset);
+      // auto old_output_offset = output_offset;
+      auto new_req =
+        req->getRequest(raw_inputs, input_offset, raw_outputs, output_offset);
       if (new_req == nullptr) {
         input_offset = old_input_offset;
-        output_offset = old_output_offset;
+        // output_offset = old_output_offset;
       } else {
         batch->addRequest(new_req);
         batch_size++;
@@ -102,6 +125,7 @@ void HardBatcher::doRun(WorkerInfo* worker) {
         batch->addTime(req->getTime());
 #endif
       }
+      first_request = false;
     } while (batch_size % this->batch_size_ != 0);
 
     if (!batch->empty()) {
diff --git a/src/amdinfer/batching/hard.hpp b/src/amdinfer/batching/hard.hpp
index e7dfd6e08..b60bb367f 100644
--- a/src/amdinfer/batching/hard.hpp
+++ b/src/amdinfer/batching/hard.hpp
@@ -24,6 +24,7 @@
 #include "amdinfer/batching/batcher.hpp"  // IWYU pragma: export
 
 namespace amdinfer {
+enum class MemoryAllocators;
 class WorkerInfo;
 }  // namespace amdinfer
 
@@ -41,7 +42,7 @@ class HardBatcher : public Batcher {
   using Batcher::Batcher;
 
  private:
-  void doRun(WorkerInfo* worker) override;
+  void doRun(const std::vector<MemoryAllocators>& allocators) override;
 };
 
 }  // namespace amdinfer
diff --git a/src/amdinfer/batching/soft.cpp b/src/amdinfer/batching/soft.cpp
index ebaac1231..64fa0a67b 100644
--- a/src/amdinfer/batching/soft.cpp
+++ b/src/amdinfer/batching/soft.cpp
@@ -29,10 +29,12 @@
 #include <utility>    // for move
 #include <vector>     // for vector
 
-#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_METRICS
-#include "amdinfer/core/exceptions.hpp"      // for invalid_argument
-#include "amdinfer/core/interface.hpp"       // for Interface
-#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/build_options.hpp"    // for AMDINFER_ENABLE_METRICS
+#include "amdinfer/core/exceptions.hpp"  // for invalid_argument
+#include "amdinfer/core/interface.hpp"   // for Interface
+#include "amdinfer/core/memory_pool/pool.hpp"
+#include "amdinfer/core/parameters.hpp"  // for ParameterMap
+#include "amdinfer/core/worker_info.hpp"
 #include "amdinfer/declarations.hpp"         // for InterfacePtr
 #include "amdinfer/observation/logging.hpp"  // for Logger, AMDINFER_LOG_DEBUG
 #include "amdinfer/observation/metrics.hpp"  // for Metrics, MetricCounterIDs
@@ -46,7 +48,7 @@ constexpr auto kDefaultTimeout = 100;
 
 namespace amdinfer {
 
-void SoftBatcher::doRun(WorkerInfo* worker) {
+void SoftBatcher::doRun(const std::vector<MemoryAllocators>& allocators) {
   auto thread_name = "batch" + this->getName();
   util::setThreadName(thread_name);
 #ifdef AMDINFER_ENABLE_LOGGING
@@ -61,13 +63,13 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
   }
 
   while (run) {
-    auto batch = std::make_unique<Batch>(worker);
-    auto input_buffers = batch->getRawInputBuffers();
-    auto output_buffers = batch->getRawOutputBuffers();
-    std::vector<size_t> input_offset(input_buffers.size(), 0);
-    std::vector<size_t> output_offset(output_buffers.size(), 0);
+    auto batch = std::make_unique<Batch>();
     size_t batch_size = 0;
 
+    std::vector<BufferPtr> input_buffers;
+    std::vector<size_t> input_offset;
+    std::vector<size_t> output_offset;
+
 #ifdef AMDINFER_ENABLE_METRICS
     Metrics::getInstance().setGauge(
       MetricGaugeIDs::QueuesBatcherInput,
@@ -106,18 +108,38 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
       }
 
       auto input_size = req->getInputSize();
-      if (input_size != input_buffers.size()) {
-        auto e =
-          invalid_argument("Number of input tensors do not match worker");
-        req->errorHandler(e);
-        continue;
-      }
       if (input_size == 0) {
         auto error = invalid_argument("Input size is zero");
         req->errorHandler(error);
         continue;
       }
 
+      if (first_request) {
+        auto input_sizes = req->getInputSizes();
+
+        input_buffers.reserve(input_sizes.size());
+        // auto output_sizes = req->getOutputSizes();
+        // TODO(varunsh): the spec does not require the request to have outputs
+        // additionally, the output size could be variable so this should be
+        // allocated by the worker
+
+        // std::vector<BufferPtr> output_buffers;
+        // output_buffers.reserve(output_sizes.size());
+        // std::vector<size_t> output_offset(output_buffers.size(), 0);
+        for (const auto& tensor_size : input_sizes) {
+          input_buffers.push_back(
+            pool_->get(allocators, tensor_size * batch_size_));
+        }
+        // for(const auto& tensor_size : output_sizes) {
+        //   output_buffers.push_back(pool_->get(allocators, tensor_size));
+        // }
+        input_offset.resize(input_buffers.size());
+        batch->setBuffers(std::move(input_buffers), {});
+      }
+
+      auto raw_inputs = batch->getRawInputBuffers();
+      auto raw_outputs = batch->getRawOutputBuffers();
+
 #ifdef AMDINFER_ENABLE_TRACING
       auto trace = req->getTrace();
       trace->startSpan("soft_batcher");
@@ -129,20 +151,17 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
 #endif
 
       auto old_input_offset = input_offset;
-      auto old_output_offset = output_offset;
-      auto new_req = req->getRequest(input_buffers, input_offset,
-                                     output_buffers, output_offset);
+      // auto old_output_offset = output_offset;
+      auto new_req =
+        req->getRequest(raw_inputs, input_offset, raw_outputs, output_offset);
       if (new_req == nullptr) {
         AMDINFER_LOG_DEBUG(logger, "Making request for " + this->model_ +
                                      " failed. Reverting buffers.");
         input_offset = old_input_offset;
-        output_offset = old_output_offset;
+        // output_offset = old_output_offset;
       } else {
         batch->addRequest(new_req);
         batch_size++;
-        if (first_request) {
-          first_request = false;
-        }
 #ifdef AMDINFER_ENABLE_TRACING
         trace->endSpan();
         batch->addTrace(std::move(trace));
@@ -151,6 +170,7 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
         batch->addTime(req->getTime());
 #endif
       }
+      first_request = false;
     } while (batch_size % this->batch_size_ != 0 && run);
 
     if (!batch->empty()) {
diff --git a/src/amdinfer/batching/soft.hpp b/src/amdinfer/batching/soft.hpp
index a1b0b4444..cf14cc86b 100644
--- a/src/amdinfer/batching/soft.hpp
+++ b/src/amdinfer/batching/soft.hpp
@@ -24,6 +24,7 @@
 #include "amdinfer/batching/batcher.hpp"  // IWYU pragma: export
 
 namespace amdinfer {
+enum class MemoryAllocators;
 class WorkerInfo;
 }  // namespace amdinfer
 
@@ -40,7 +41,7 @@ class SoftBatcher : public Batcher {
   using Batcher::Batcher;
 
  private:
-  void doRun(WorkerInfo* worker) override;
+  void doRun(const std::vector<MemoryAllocators>& allocators) override;
 };
 
 }  // namespace amdinfer
diff --git a/src/amdinfer/buffers/CMakeLists.txt b/src/amdinfer/buffers/CMakeLists.txt
index 02125b9af..dd4b8c9e5 100644
--- a/src/amdinfer/buffers/CMakeLists.txt
+++ b/src/amdinfer/buffers/CMakeLists.txt
@@ -13,21 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-list(APPEND targets buffer vector_buffer)
+list(APPEND targets cpu)
 
-if(${AMDINFER_ENABLE_VITIS})
-  set(targets ${targets} vart_tensor_buffer)
-endif()
+add_library(buffer OBJECT buffer.cpp)
+target_include_directories(buffer PRIVATE ${AMDINFER_INCLUDE_DIRS})
+set_target_options(buffer)
+
+list(APPEND BUFFERS_OBJS $<TARGET_OBJECTS:buffer>)
 
 foreach(target ${targets})
-  add_library(${target} OBJECT ${target}.cpp)
-  target_include_directories(${target} PRIVATE ${AMDINFER_INCLUDE_DIRS})
-  set_target_options(${target})
+  add_library(${target}Buffer OBJECT ${target}.cpp)
+  target_include_directories(${target}Buffer PRIVATE ${AMDINFER_INCLUDE_DIRS})
+  set_target_options(${target}Buffer)
 
-  list(APPEND BUFFERS_OBJS $<TARGET_OBJECTS:${target}>)
+  list(APPEND BUFFERS_OBJS $<TARGET_OBJECTS:${target}Buffer>)
 endforeach()
 
 add_library(buffers STATIC ${BUFFERS_OBJS})
 set_target_options(buffers)
 
-# set(BUFFERS_OBJS ${BUFFERS_OBJS} PARENT_SCOPE)
+set(BUFFERS_OBJS ${BUFFERS_OBJS} PARENT_SCOPE)
diff --git a/src/amdinfer/buffers/buffer.hpp b/src/amdinfer/buffers/buffer.hpp
index d6cad4a9a..f3476c5d7 100644
--- a/src/amdinfer/buffers/buffer.hpp
+++ b/src/amdinfer/buffers/buffer.hpp
@@ -51,12 +51,6 @@ class Buffer {
    */
   virtual void* data(size_t offset) = 0;
 
-  /**
-   * @brief Reset the buffer. This should be called prior to returning the
-   * buffer to its buffer pool.
-   */
-  virtual void reset() = 0;
-
   /**
    * @brief Write arbitrary data from an address into this buffer.
    *
@@ -68,7 +62,7 @@ class Buffer {
   virtual size_t write(void* data, size_t offset, size_t size);
 
   /**
-   * @brief Write a bool value to the buffer
+   * @brief Write a value to the buffer
    *
    * @param value value to write
    */
@@ -89,8 +83,7 @@ class Buffer {
       // this->write_counter_ += value.length() + 1;
       return offset + value.length() + 1;
     } else {
-      std::memcpy(this->data(offset), &value, sizeof(T));
-      return offset + sizeof(T);
+      return this->write(&value, offset, sizeof(T));
     }
   }
 };
diff --git a/src/amdinfer/buffers/cpu.cpp b/src/amdinfer/buffers/cpu.cpp
new file mode 100644
index 000000000..a99d6d0bb
--- /dev/null
+++ b/src/amdinfer/buffers/cpu.cpp
@@ -0,0 +1,33 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief Implements the CpuBuffer class
+ */
+
+#include "amdinfer/buffers/cpu.hpp"
+
+#include <algorithm>  // for max
+#include <memory>     // for make_unique
+#include <utility>    // for move
+
+namespace amdinfer {
+
+CpuBuffer::CpuBuffer(void* data, MemoryAllocators allocator)
+  : allocator_(allocator), data_(static_cast<std::byte*>(data)) {}
+
+void* CpuBuffer::data(size_t offset) { return data_ + offset; }
+
+}  // namespace amdinfer
diff --git a/src/amdinfer/buffers/cpu.hpp b/src/amdinfer/buffers/cpu.hpp
new file mode 100644
index 000000000..438b0513e
--- /dev/null
+++ b/src/amdinfer/buffers/cpu.hpp
@@ -0,0 +1,63 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#ifndef GUARD_AMDINFER_BUFFERS_CPU
+#define GUARD_AMDINFER_BUFFERS_CPU
+
+#include <cstddef>  // for size_t, byte
+#include <vector>   // for vector
+
+#include "amdinfer/buffers/buffer.hpp"   // IWYU pragma: export
+#include "amdinfer/core/data_types.hpp"  // for DataType
+#include "amdinfer/core/memory_pool/pool.hpp"
+#include "amdinfer/util/queue.hpp"  // for BufferPtrsQueue
+
+namespace amdinfer {
+
+/**
+ * @brief VectorBuffer uses vectors for storing data
+ *
+ */
+class CpuBuffer : public Buffer {
+ public:
+  /**
+   * @brief Construct a new Vector Buffer object
+   *
+   * @param elements number of elements to store
+   * @param data_type type of element to store
+   */
+  CpuBuffer(void* data, MemoryAllocators allocator);
+
+  /**
+   * @brief Returns a pointer to the underlying data
+   *
+   * @return void*
+   */
+  void* data(size_t offset) override;
+
+  MemoryAllocators getAllocator() const { return allocator_; }
+
+ private:
+  MemoryAllocators allocator_;
+  std::byte* data_;
+};
+
+}  // namespace amdinfer
+
+#endif  // GUARD_AMDINFER_BUFFERS_CPU
diff --git a/src/amdinfer/buffers/vart_tensor_buffer.cpp b/src/amdinfer/buffers/vart_tensor_buffer.cpp
deleted file mode 100644
index 43d7fdb1c..000000000
--- a/src/amdinfer/buffers/vart_tensor_buffer.cpp
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright 2021 Xilinx, Inc.
-// Copyright 2022 Advanced Micro Devices, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * @file
- * @brief Implements the VartTensorBuffer class
- */
-
-#include "amdinfer/buffers/vart_tensor_buffer.hpp"
-
-#include <utility>  // for pair
-
-namespace amdinfer {
-
-VartTensorBuffer::VartTensorBuffer(const std::string& name,
-                                   std::vector<int32_t>& shape,
-                                   xir::DataType data_type)
-  : tensor_(xir::Tensor::create(name, shape, data_type)),
-    data_(tensor_.get()) {}
-
-VartTensorBuffer::VartTensorBuffer(const char* name,
-                                   std::vector<int32_t>& shape,
-                                   xir::DataType data_type)
-  : tensor_(xir::Tensor::create(name, shape, data_type)),
-    data_(tensor_.get()) {}
-
-void* VartTensorBuffer::data(size_t offset) {
-  // Some DPUs need a shape argument to data() to get the data properly.
-  // This argument should be the same size as the tensor (by default,
-  // [batch, h, w, c]). This first argument is the batch index. The other
-  // indices should be zero to get the start of the batch
-  std::vector<int> shape(this->tensor_->get_shape().size(), 0);
-
-  auto dims = this->tensor_->get_shape();
-  auto size = this->tensor_->get_shape().size();
-
-  // convert the offset to a shape based on the tensor shape
-  for (auto k = 0U; k < size; k++) {
-    auto stride = 1;
-    for (auto m = k + 1; m < size; m++) {
-      stride *= dims[m];
-    }
-    shape[k] = offset / stride;
-    offset -= shape[k] * stride;
-  }
-
-  return reinterpret_cast<void*>(this->data_.data(shape).first);
-}
-
-void VartTensorBuffer::reset() {}
-
-vart::TensorBuffer* VartTensorBuffer::getTensorBuffer() { return &this->data_; }
-
-}  // namespace amdinfer
diff --git a/src/amdinfer/buffers/vart_tensor_buffer.hpp b/src/amdinfer/buffers/vart_tensor_buffer.hpp
deleted file mode 100644
index 015d654c8..000000000
--- a/src/amdinfer/buffers/vart_tensor_buffer.hpp
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2021 Xilinx, Inc.
-// Copyright 2022 Advanced Micro Devices, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * @file
- * @brief Defines the VartTensorBuffer class
- */
-
-#ifndef GUARD_AMDINFER_BUFFERS_VART_TENSOR_BUFFER
-#define GUARD_AMDINFER_BUFFERS_VART_TENSOR_BUFFER
-
-#include <cstddef>                              // for size_t
-#include <cstdint>                              // for int32_t
-#include <memory>                               // for unique_ptr
-#include <string>                               // for string
-#include <vart/experimental/runner_helper.hpp>  // for CpuFlatTensorBufferOwned
-#include <vector>                               // for vector
-#include <xir/tensor/tensor.hpp>                // for Tensor
-#include <xir/util/data_type.hpp>               // for DataType
-
-#include "amdinfer/buffers/buffer.hpp"
-
-namespace vart {
-class TensorBuffer;
-}  // namespace vart
-
-namespace amdinfer {
-
-/**
- * @brief VartTensorBuffer uses vart::CpuFlatTensorBufferOwned for storing data
- *
- */
-class VartTensorBuffer : public Buffer {
- public:
-  VartTensorBuffer(const std::string& name, std::vector<int32_t>& shape,
-                   xir::DataType data_type);
-  VartTensorBuffer(const char* name, std::vector<int32_t>& shape,
-                   xir::DataType data_type);
-
-  /**
-   * @brief Returns a pointer to the underlying data.
-   *
-   * @param index Used to index to the correct tensor in the batch
-   *
-   * @return void*
-   */
-  void* data(size_t offset) override;
-
-  /**
-   * @brief Perform any cleanup before returning the buffer to the pool
-   */
-  void reset() override;
-
-  /// Get a pointer to the underlying TensorBuffer
-  vart::TensorBuffer* getTensorBuffer();
-
- private:
-  std::unique_ptr<xir::Tensor> tensor_;
-  vart::CpuFlatTensorBufferOwned data_;
-};
-
-}  // namespace amdinfer
-
-#endif  // GUARD_AMDINFER_BUFFERS_VART_TENSOR_BUFFER
diff --git a/src/amdinfer/buffers/vector_buffer.cpp b/src/amdinfer/buffers/vector_buffer.cpp
deleted file mode 100644
index 1df44ddbc..000000000
--- a/src/amdinfer/buffers/vector_buffer.cpp
+++ /dev/null
@@ -1,181 +0,0 @@
-// Copyright 2021 Xilinx, Inc.
-// Copyright 2022 Advanced Micro Devices, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * @file
- * @brief Implements the VectorBuffer class
- */
-
-#include "amdinfer/buffers/vector_buffer.hpp"
-
-#include <algorithm>  // for max
-#include <memory>     // for make_unique
-#include <utility>    // for move
-
-#include "amdinfer/declarations.hpp"  // for BufferPtrs
-
-namespace amdinfer {
-
-VectorBuffer::VectorBuffer(int elements, DataType data_type)
-  : type_(data_type) {
-  std::size_t size = elements * data_type.size();
-  this->data_.reserve(size);
-  // this->write_counter_ = 0;
-}
-
-void* VectorBuffer::data(size_t offset) {
-  return static_cast<std::byte*>(this->data_.data()) + offset;
-}
-void VectorBuffer::reset() {}
-
-// size_t VectorBuffer::write(bool value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Bool);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(uint8_t value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Uint8);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(uint16_t value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Uint16);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(uint32_t value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Uint32);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(uint64_t value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Uint64);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(int8_t value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Int8);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(int16_t value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Int16);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(int32_t value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Int32);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(int64_t value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Int64);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(float value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Fp32);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// size_t VectorBuffer::write(double value, size_t offset) {
-//   constexpr auto bytes = types::getSize(DataType::Fp64);
-//   std::memcpy(this->data_.data() + offset, &value, bytes);
-//   return offset + bytes;
-// }
-// // not quite sure what the best way to do this is. The commented out code is
-// // other attempts. This works but may be non-optimal.
-// size_t VectorBuffer::write(std::string value, size_t offset) {
-//   // constexpr auto bytes = types::getSize(DataType::String);
-//   // std::memcpy(this->data_.data() + offset, &value, bytes);
-
-//   char null_term = '\0';
-//   std::copy(value.begin(), value.end(),
-//             reinterpret_cast<char*>(this->data_.data() + offset));
-//   // strcpy(reinterpret_cast<char*>(this->data_.data() +
-//   // this->write_counter_),
-//   // value.c_str());
-//   std::memcpy(this->data_.data() + offset + value.length(), &null_term, 1);
-
-//   // this->write_counter_ += value.length() + 1;
-//   return offset + value.length() + 1;
-// }
-
-// Status VectorBuffer::read(int index, bool& value) {
-//   value = *reinterpret_cast<bool*>(
-//     &this->data_[index * types::getSize(DataType::Bool)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, uint8_t& value) {
-//   value = *reinterpret_cast<uint8_t*>(
-//     &this->data_[index * types::getSize(DataType::Uint8)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, uint16_t& value) {
-//   value = *reinterpret_cast<uint16_t*>(
-//     &this->data_[index * types::getSize(DataType::Uint16)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, uint32_t& value) {
-//   value = *reinterpret_cast<uint32_t*>(
-//     &this->data_[index * types::getSize(DataType::Uint32)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, uint64_t& value) {
-//   value = *reinterpret_cast<uint64_t*>(
-//     &this->data_[index * types::getSize(DataType::Uint64)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, int8_t& value) {
-//   value = *reinterpret_cast<int8_t*>(
-//     &this->data_[index * types::getSize(DataType::Int8)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, int16_t& value) {
-//   value = *reinterpret_cast<int16_t*>(
-//     &this->data_[index * types::getSize(DataType::Int16)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, int32_t& value) {
-//   value = *reinterpret_cast<int32_t*>(
-//     &this->data_[index * types::getSize(DataType::Int32)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, int64_t& value) {
-//   value = *reinterpret_cast<int64_t*>(
-//     &this->data_[index * types::getSize(DataType::Int64)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, float& value) {
-//   value = *reinterpret_cast<float*>(
-//     &this->data_[index * types::getSize(DataType::Fp32)]);
-//   return Status::kOK;
-// }
-// Status VectorBuffer::read(int index, double& value) {
-//   value = *reinterpret_cast<double*>(
-//     &this->data_[index * types::getSize(DataType::Fp64)]);
-//   return Status::kOK;
-// }
-
-void VectorBuffer::allocate(BufferPtrsQueue* my_buffer, size_t num,
-                            size_t elements, DataType data_type) {
-  for (size_t i = 0; i < num; i++) {
-    BufferPtrs vec;
-    vec.emplace_back(std::make_unique<VectorBuffer>(elements, data_type));
-    my_buffer->enqueue(std::move(vec));
-  }
-}
-
-}  // namespace amdinfer
diff --git a/src/amdinfer/buffers/vector_buffer.hpp b/src/amdinfer/buffers/vector_buffer.hpp
deleted file mode 100644
index 36716d9f2..000000000
--- a/src/amdinfer/buffers/vector_buffer.hpp
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2021 Xilinx, Inc.
-// Copyright 2022 Advanced Micro Devices, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * @file
- * @brief Defines the VectorBuffer class
- */
-
-#ifndef GUARD_AMDINFER_BUFFERS_VECTOR_BUFFER
-#define GUARD_AMDINFER_BUFFERS_VECTOR_BUFFER
-
-#include <cstddef>  // for size_t, byte
-#include <vector>   // for vector
-
-#include "amdinfer/buffers/buffer.hpp"   // IWYU pragma: export
-#include "amdinfer/core/data_types.hpp"  // for DataType
-#include "amdinfer/util/queue.hpp"       // for BufferPtrsQueue
-
-namespace amdinfer {
-
-/**
- * @brief VectorBuffer uses vectors for storing data
- *
- */
-class VectorBuffer : public Buffer {
- public:
-  /**
-   * @brief Construct a new Vector Buffer object
-   *
-   * @param elements number of elements to store
-   * @param data_type type of element to store
-   */
-  VectorBuffer(int elements, DataType data_type);
-
-  /**
-   * @brief Returns a pointer to the underlying data. Vectors guarantee that it
-   * is contiguous.
-   *
-   * @return void*
-   */
-  void* data(size_t offset) override;
-
-  /**
-   * @brief Reset the internal write_counter_ prior to returning the buffer to
-   * a pool
-   */
-  void reset() override;
-
-  /**
-   * @brief Allocate VectorBuffers in the given BufferPtrsQueue
-   *
-   * @param my_buffer a queue of Buffer pointers
-   * @param num number of VectorBuffers to allocate
-   * @param elements number of elements per VectorBuffer
-   * @param data_type data type of the VectorBuffer (used for sizing)
-   */
-  static void allocate(BufferPtrsQueue* my_buffer, size_t num, size_t elements,
-                       DataType data_type);
-
- private:
-  /// The type of the data is used to size the vector appropriately
-  DataType type_;
-  /// Our actual data is allocated as bytes to support different types
-  std::vector<std::byte> data_;
-  /// Counter to track the next index to write at
-  // size_t write_counter_ = 0;
-};
-
-}  // namespace amdinfer
-
-#endif  // GUARD_AMDINFER_BUFFERS_VECTOR_BUFFER
diff --git a/src/amdinfer/clients/http_internal.cpp b/src/amdinfer/clients/http_internal.cpp
index 97edab87b..b34e53108 100644
--- a/src/amdinfer/clients/http_internal.cpp
+++ b/src/amdinfer/clients/http_internal.cpp
@@ -458,12 +458,38 @@ size_t DrogonHttp::getInputSize() {
   }
   auto inputs = this->json_->get("inputs", Json::arrayValue);
   if (!inputs.isArray()) {
-    throw invalid_argument("'inputs' is not an arrafy");
+    throw invalid_argument("'inputs' is not an array");
   }
 
   return inputs.size();
 }
 
+std::vector<size_t> DrogonHttp::getInputSizes() const {
+  std::vector<size_t> sizes;
+
+  if (!this->json_->isMember("inputs")) {
+    throw invalid_argument("No 'inputs' key present in request");
+  }
+  auto inputs = this->json_->get("inputs", Json::arrayValue);
+  if (!inputs.isArray()) {
+    throw invalid_argument("'inputs' is not an array");
+  }
+
+  for (const auto &tensor : inputs) {
+    // using asCString() doesn't work -> the string is empty
+    const auto raw_type = tensor.get("datatype", "UNKNOWN").asString();
+    auto datatype = DataType(raw_type.c_str());
+
+    const auto shape = tensor.get("shape", Json::arrayValue);
+    size_t size = 1;
+    for (const auto &index : shape) {
+      size *= index.asInt64();
+    }
+    sizes.push_back(size * datatype.size());
+  }
+  return sizes;
+}
+
 Json::Value parseResponse(InferenceResponse response) {
   Json::Value ret;
   ret["model_name"] = response.getModel();
diff --git a/src/amdinfer/clients/http_internal.hpp b/src/amdinfer/clients/http_internal.hpp
index 2a9882d2a..8b66494ad 100644
--- a/src/amdinfer/clients/http_internal.hpp
+++ b/src/amdinfer/clients/http_internal.hpp
@@ -97,6 +97,7 @@ class DrogonHttp : public Interface {
     std::vector<size_t> &output_offsets) override;
 
   size_t getInputSize() override;
+  std::vector<size_t> getInputSizes() const override;
   void errorHandler(const std::exception &e) override;
 
  private:
diff --git a/src/amdinfer/clients/native_internal.cpp b/src/amdinfer/clients/native_internal.cpp
index 4832745c6..c22ca4dd7 100644
--- a/src/amdinfer/clients/native_internal.cpp
+++ b/src/amdinfer/clients/native_internal.cpp
@@ -137,6 +137,22 @@ CppNativeApi::CppNativeApi(InferenceRequest request)
 
 size_t CppNativeApi::getInputSize() { return this->request_.getInputSize(); }
 
+std::vector<size_t> CppNativeApi::getInputSizes() const {
+  std::vector<size_t> sizes;
+  for (const auto &tensor : request_.getInputs()) {
+    sizes.push_back(tensor.getSize() * tensor.getDatatype().size());
+  }
+  return sizes;
+}
+
+// std::vector<size_t> CppNativeApi::getOutputSizes() {
+//   std::vector<size_t> sizes;
+//   for(const auto& tensor : request_.getOutputs()){
+//     sizes.push_back(tensor.getSize() * tensor.getDatatype().size);
+//   }
+//   return sizes;
+// }
+
 std::promise<amdinfer::InferenceResponse> *CppNativeApi::getPromise() {
   return this->promise_.get();
 }
diff --git a/src/amdinfer/clients/native_internal.hpp b/src/amdinfer/clients/native_internal.hpp
index 9bc69e02a..99806a32d 100644
--- a/src/amdinfer/clients/native_internal.hpp
+++ b/src/amdinfer/clients/native_internal.hpp
@@ -43,6 +43,8 @@ class CppNativeApi : public Interface {
     std::vector<size_t> &output_offsets) override;
 
   size_t getInputSize() override;
+  std::vector<size_t> getInputSizes() const override;
+  // std::vector<size_t> getOutputSizes() override;
   void errorHandler(const std::exception &e) override;
   std::promise<amdinfer::InferenceResponse> *getPromise();
 
diff --git a/src/amdinfer/core/CMakeLists.txt b/src/amdinfer/core/CMakeLists.txt
index b0c9466a0..6d041647e 100644
--- a/src/amdinfer/core/CMakeLists.txt
+++ b/src/amdinfer/core/CMakeLists.txt
@@ -73,8 +73,10 @@ target_include_directories(
 target_link_libraries(model_repository INTERFACE lib_config efsw)
 add_dependencies(model_repository lib_config)
 
-add_library(core STATIC ${CORE_OBJS})
-target_link_libraries(core INTERFACE ${targets})
+add_subdirectory(memory_pool) # sets MEMORY_POOL
+
+add_library(core STATIC ${CORE_OBJS} ${MEMORY_POOL})
+target_link_libraries(core INTERFACE ${targets} memory_pool)
 set_target_options(core)
 
 set(CORE_OBJS ${CORE_OBJS} PARENT_SCOPE)
diff --git a/src/amdinfer/core/endpoints.cpp b/src/amdinfer/core/endpoints.cpp
index 642034a87..ce85ae55b 100644
--- a/src/amdinfer/core/endpoints.cpp
+++ b/src/amdinfer/core/endpoints.cpp
@@ -237,12 +237,13 @@ std::string Endpoints::unsafeLoad(const std::string& worker,
   // if the worker doesn't exist yet, we need to create it
   try {
     if (worker_info == nullptr) {
-      auto new_worker = std::make_unique<WorkerInfo>(worker_name, parameters);
+      auto new_worker =
+        std::make_unique<WorkerInfo>(worker_name, parameters, &pool_);
       this->workers_.try_emplace(endpoint, std::move(new_worker));
       // if the worker exists but the share parameter is false, we need to add
       // one
     } else if (!share) {
-      worker_info->addAndStartWorker(worker_name, parameters);
+      worker_info->addAndStartWorker(worker_name, parameters, &pool_);
     }
   } catch (...) {
     // undo the load if the worker creation fails
diff --git a/src/amdinfer/core/endpoints.hpp b/src/amdinfer/core/endpoints.hpp
index 9321de882..7e9d39288 100644
--- a/src/amdinfer/core/endpoints.hpp
+++ b/src/amdinfer/core/endpoints.hpp
@@ -29,10 +29,11 @@
 #include <utility>        // for move
 #include <vector>         // for vector
 
-#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE...
-#include "amdinfer/core/predict_api.hpp"     // for ModelMetadata
-#include "amdinfer/observation/logging.hpp"  // for Logger, Loggers
-#include "amdinfer/util/queue.hpp"           // for BlockingQueue
+#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE...
+#include "amdinfer/core/memory_pool/pool.hpp"  // for MemoryPool
+#include "amdinfer/core/predict_api.hpp"       // for ModelMetadata
+#include "amdinfer/observation/logging.hpp"    // for Logger, Loggers
+#include "amdinfer/util/queue.hpp"             // for BlockingQueue
 
 namespace amdinfer {
 
@@ -113,6 +114,7 @@ class Endpoints {
   /// A queue used to sequentially order changes to the Manager state
   UpdateCommandQueue update_queue_;
   std::thread update_thread_;
+  MemoryPool pool_;
 #ifdef AMDINFER_ENABLE_LOGGING
   Logger logger_{Loggers::Server};
 #endif
diff --git a/src/amdinfer/core/interface.hpp b/src/amdinfer/core/interface.hpp
index 709f58d02..54f5262dd 100644
--- a/src/amdinfer/core/interface.hpp
+++ b/src/amdinfer/core/interface.hpp
@@ -78,6 +78,9 @@ class Interface {
 #endif
   /// Get the number of input tensors in the request
   virtual size_t getInputSize() = 0;
+
+  virtual std::vector<size_t> getInputSizes() const = 0;
+  // virtual std::vector<size_t> getOutputSizes() = 0;
   /**
    * @brief Construct an InferenceRequest using the data in the Interface
    *
diff --git a/src/amdinfer/core/memory_pool/CMakeLists.txt b/src/amdinfer/core/memory_pool/CMakeLists.txt
new file mode 100644
index 000000000..e3f0b9607
--- /dev/null
+++ b/src/amdinfer/core/memory_pool/CMakeLists.txt
@@ -0,0 +1,30 @@
+# Copyright 2021 Xilinx, Inc.
+# Copyright 2022 Advanced Micro Devices, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(targets cpu_allocator pool)
+
+foreach(target ${targets})
+  add_library(${target} OBJECT ${target}.cpp)
+  target_include_directories(${target} PRIVATE ${AMDINFER_INCLUDE_DIRS})
+  set_target_options(${target})
+
+  list(APPEND MEMORY_POOL_OBJS $<TARGET_OBJECTS:${target}>)
+endforeach()
+
+add_library(memory_pool STATIC ${MEMORY_POOL_OBJS})
+target_link_libraries(memory_pool INTERFACE ${targets})
+set_target_options(memory_pool)
+
+set(MEMORY_POOL ${MEMORY_POOL_OBJS} PARENT_SCOPE)
diff --git a/src/amdinfer/core/memory_pool/cpu_allocator.cpp b/src/amdinfer/core/memory_pool/cpu_allocator.cpp
new file mode 100644
index 000000000..4b55bf15f
--- /dev/null
+++ b/src/amdinfer/core/memory_pool/cpu_allocator.cpp
@@ -0,0 +1,118 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#include "amdinfer/core/memory_pool/cpu_allocator.hpp"
+
+#include <cassert>
+#include <iostream>
+#include <vector>
+
+#include "amdinfer/core/exceptions.hpp"
+
+namespace amdinfer {
+
+CpuAllocator::CpuAllocator(size_t block_size, size_t max_allocate)
+  : max_allocate_(max_allocate), block_size_(block_size) {}
+
+void* CpuAllocator::get(size_t size) {
+  const std::lock_guard lock{mutex_};
+  auto best = headers_.end();
+  const auto end = headers_.end();
+  for (auto it = headers_.begin(); it != end; it++) {
+    // if this block can hold this data and it's smaller than our current best
+    if (it->free && it->size >= size &&
+        (best == end || it->size < best->size)) {
+      best = it;
+    }
+  }
+
+  if (best != end) {
+    if (best->size == size) {
+      best->free = false;
+      // std::cout << "Matched " << size << " bytes\n";
+      return best->address;
+    }
+    const auto& new_block =
+      headers_.emplace(best, best->address, size, false, best->block_id);
+    assert(end == headers_.end());
+    best->size -= size;
+    best->address += size;
+    // std::cout << "Partitioned " << size << " bytes\n";
+    return new_block->address;
+  }
+
+  auto size_to_allocate = std::max(size, block_size_);
+  if (allocated_ + size_to_allocate > max_allocate_) {
+    throw runtime_error("Too much requested");
+  }
+  auto& new_block = data_.emplace_back();
+  new_block.resize(size_to_allocate);
+  allocated_ += size_to_allocate;
+
+  auto* retval = new_block.data();
+  block_id_++;
+
+  headers_.emplace_back(retval, size, false, block_id_);
+  assert(end == headers_.end());
+  if (size < block_size_) {
+    headers_.emplace_back(retval + size, block_size_ - size, true, block_id_);
+    assert(end == headers_.end());
+    // foo.next = std::prev(end);
+  }
+
+  // std::cout << "Allocated " << size << " bytes\n";
+  return retval;
+}
+
+void CpuAllocator::put(const void* address) {
+  const std::lock_guard lock{mutex_};
+  const auto end = headers_.end();
+  auto found = headers_.end();
+  for (auto it = headers_.begin(); it != end; it++) {
+    if (it->address == address) {
+      found = it;
+      break;
+    }
+  }
+
+  if (found == end) {
+    throw runtime_error("Address not found");
+  }
+
+  // if the previous is free and from the same block, merge the two
+  if (auto prev = std::prev(found);
+      prev->block_id == found->block_id && prev->free) {
+    prev->size += found->size;
+    headers_.erase(found);
+    found = prev;
+  }
+
+  // if the next is free and from the same block, merge the two
+  if (auto next = std::next(found);
+      next->block_id == found->block_id && next->free) {
+    next->size += found->size;
+    next->address = found->address;
+    headers_.erase(found);
+  } else {
+    found->free = true;
+  }
+  // std::cout << "Freed memory\n";
+}
+
+}  // namespace amdinfer
diff --git a/src/amdinfer/core/memory_pool/cpu_allocator.hpp b/src/amdinfer/core/memory_pool/cpu_allocator.hpp
new file mode 100644
index 000000000..15b98297f
--- /dev/null
+++ b/src/amdinfer/core/memory_pool/cpu_allocator.hpp
@@ -0,0 +1,55 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#ifndef GUARD_AMDINFER_CORE_MEMORY_POOL_CPU_ALLOCATOR
+#define GUARD_AMDINFER_CORE_MEMORY_POOL_CPU_ALLOCATOR
+
+#include <cstddef>
+#include <list>
+#include <mutex>
+#include <vector>
+
+#include "amdinfer/core/memory_pool/memory_allocator.hpp"
+
+namespace amdinfer {
+
+struct MemoryHeader;
+
+class CpuAllocator : public MemoryAllocator {
+ public:
+  explicit CpuAllocator(size_t block_size, size_t max_allocated = -1);
+
+  [[nodiscard]] void* get(size_t size) override;
+  void put(const void* address) override;
+
+  // void free(const void* address);
+
+ private:
+  size_t allocated_ = 0;
+  size_t max_allocate_;
+  size_t block_size_;
+  size_t block_id_ = 0;
+  std::mutex mutex_;
+  std::list<MemoryHeader> headers_;
+  std::vector<std::vector<std::byte>> data_;
+};
+
+}  // namespace amdinfer
+
+#endif  // GUARD_AMDINFER_CORE_MEMORY_POOL_CPU_ALLOCATOR
diff --git a/src/amdinfer/core/memory_pool/memory_allocator.hpp b/src/amdinfer/core/memory_pool/memory_allocator.hpp
new file mode 100644
index 000000000..df3857f2f
--- /dev/null
+++ b/src/amdinfer/core/memory_pool/memory_allocator.hpp
@@ -0,0 +1,48 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#ifndef GUARD_AMDINFER_CORE_MEMORY_POOL_MEMORY_ALLOCATOR
+#define GUARD_AMDINFER_CORE_MEMORY_POOL_MEMORY_ALLOCATOR
+
+#include <cstddef>  // for size_t
+
+namespace amdinfer {
+
+struct MemoryHeader {
+  std::byte* address;
+  bool free;
+  size_t size;
+  size_t block_id;
+
+  MemoryHeader(std::byte* address, size_t size, bool free, size_t block_id)
+    : address(address), free(free), size(size), block_id(block_id) {}
+};
+
+class MemoryAllocator {
+ public:
+  virtual ~MemoryAllocator() = default;
+
+  // these methods are thread-safe
+  [[nodiscard]] virtual void* get(size_t size) = 0;
+  virtual void put(const void* address) = 0;
+};
+
+}  // namespace amdinfer
+
+#endif  // GUARD_AMDINFER_CORE_MEMORY_POOL_MEMORY_ALLOCATOR
diff --git a/src/amdinfer/core/memory_pool/pool.cpp b/src/amdinfer/core/memory_pool/pool.cpp
new file mode 100644
index 000000000..e80b6625e
--- /dev/null
+++ b/src/amdinfer/core/memory_pool/pool.cpp
@@ -0,0 +1,58 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#include "amdinfer/core/memory_pool/pool.hpp"
+
+#include "amdinfer/buffers/cpu.hpp"
+#include "amdinfer/core/exceptions.hpp"
+#include "amdinfer/core/memory_pool/cpu_allocator.hpp"
+
+namespace amdinfer {
+
+const size_t kDefaultCpuBlockSize = 1'048'576;  // arbitrarily 1MiB
+
+MemoryPool::MemoryPool() {
+  allocators_.try_emplace(MemoryAllocators::Cpu,
+                          std::make_unique<CpuAllocator>(kDefaultCpuBlockSize));
+}
+
+std::unique_ptr<Buffer> MemoryPool::get(
+  const std::vector<MemoryAllocators>& allocators, size_t size) {
+  for (const auto& allocator : allocators) {
+    void* address = nullptr;
+    try {
+      address = allocators_.at(allocator)->get(size);
+    } catch (const runtime_error&) {
+      continue;
+    }
+    if (address != nullptr) {
+      return std::make_unique<CpuBuffer>(address, allocator);
+    }
+  }
+  throw runtime_error("Memory could not be allocated");
+}
+
+void MemoryPool::put(std::unique_ptr<Buffer> memory) {
+  auto* buffer = dynamic_cast<CpuBuffer*>(memory.get());
+  const auto allocator = buffer->getAllocator();
+  const auto* address = buffer->data(0);
+  allocators_.at(allocator)->put(address);
+}
+
+}  // namespace amdinfer
diff --git a/src/amdinfer/core/memory_pool/pool.hpp b/src/amdinfer/core/memory_pool/pool.hpp
new file mode 100644
index 000000000..bd16d73f5
--- /dev/null
+++ b/src/amdinfer/core/memory_pool/pool.hpp
@@ -0,0 +1,53 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#ifndef GUARD_AMDINFER_CORE_MEMORY_POOL_POOL
+#define GUARD_AMDINFER_CORE_MEMORY_POOL_POOL
+
+#include <cstddef>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "amdinfer/buffers/buffer.hpp"
+#include "amdinfer/build_options.hpp"
+#include "amdinfer/core/memory_pool/memory_allocator.hpp"
+
+namespace amdinfer {
+
+enum class MemoryAllocators { Cpu };
+
+using Memory = std::pair<MemoryAllocators, void*>;
+
+class MemoryPool {
+ public:
+  MemoryPool();
+
+  std::unique_ptr<Buffer> get(const std::vector<MemoryAllocators>& allocators,
+                              size_t size);
+  void put(std::unique_ptr<Buffer> memory);
+
+ private:
+  std::unordered_map<MemoryAllocators, std::unique_ptr<MemoryAllocator>>
+    allocators_;
+};
+
+}  // namespace amdinfer
+
+#endif  // GUARD_AMDINFER_CORE_MEMORY_POOL_POOL
diff --git a/src/amdinfer/core/worker_info.cpp b/src/amdinfer/core/worker_info.cpp
index 0fa53d411..9f1687133 100644
--- a/src/amdinfer/core/worker_info.cpp
+++ b/src/amdinfer/core/worker_info.cpp
@@ -34,9 +34,10 @@
 #include "amdinfer/batching/batcher.hpp"  // for Batcher, BatcherStatus, Bat...
 #include "amdinfer/core/exceptions.hpp"   // for invalid_argument, external_...
 #include "amdinfer/core/interface.hpp"    // IWYU pragma: keep
-#include "amdinfer/core/parameters.hpp"   // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"  // for ModelMetadata
-#include "amdinfer/workers/worker.hpp"    // for Worker, WorkerStatus, Worke...
+#include "amdinfer/core/memory_pool/pool.hpp"  // for MemoryPool
+#include "amdinfer/core/parameters.hpp"        // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"       // for ModelMetadata
+#include "amdinfer/workers/worker.hpp"  // for Worker, WorkerStatus, Worke...
 
 namespace amdinfer {
 
@@ -104,11 +105,9 @@ workers::Worker* getWorker(const std::string& name) {
   return worker;
 }
 
-WorkerInfo::WorkerInfo(const std::string& name, ParameterMap* parameters) {
-  this->input_buffer_ptr_ = std::make_unique<BufferPtrsQueue>();
-  this->output_buffer_ptr_ = std::make_unique<BufferPtrsQueue>();
-
-  this->addAndStartWorker(name, parameters);
+WorkerInfo::WorkerInfo(const std::string& name, ParameterMap* parameters,
+                       MemoryPool* pool) {
+  this->addAndStartWorker(name, parameters, pool);
 }
 
 WorkerInfo::~WorkerInfo() {
@@ -119,23 +118,12 @@ WorkerInfo::~WorkerInfo() {
 }
 
 void WorkerInfo::addAndStartWorker(const std::string& name,
-                                   ParameterMap* parameters) {
+                                   ParameterMap* parameters, MemoryPool* pool) {
   auto* worker = getWorker(name);
   worker->init(parameters);
 
-  auto max_buffers = worker->getMaxBufferNum();
-  worker->setInputBuffers(this->input_buffer_ptr_.get());
-  worker->setOutputBuffers(this->output_buffer_ptr_.get());
-  try {
-    auto buffer_num = worker->allocate(kNumBufferAuto);
-    this->buffer_num_ += buffer_num;
-  } catch (const std::exception& e) {
-    throw external_error(e.what());
-  } catch (...) {
-    throw runtime_error("Unknown error occurred");
-  }
-  this->max_buffer_num_ =
-    max_buffers == UINT_MAX ? UINT_MAX : this->max_buffer_num_ + max_buffers;
+  std::vector<MemoryAllocators> allocators = worker->getAllocators();
+  ;
 
   try {
     worker->acquire(parameters);
@@ -146,13 +134,14 @@ void WorkerInfo::addAndStartWorker(const std::string& name,
   }
 
   this->batch_size_ = worker->getBatchSize();
+  worker->setPool(pool);
 
   if (this->batchers_.empty()) {
     int32_t batcher_count = 1;
     if (parameters->has("batchers")) {
       batcher_count = parameters->get<int32_t>("batchers");
     }
-    this->batchers_ = worker->makeBatcher(batcher_count, parameters);
+    this->batchers_ = worker->makeBatcher(batcher_count, parameters, pool);
 
     for (const auto& batcher : this->batchers_) {
       batcher->setName(name);
@@ -162,7 +151,7 @@ void WorkerInfo::addAndStartWorker(const std::string& name,
 
   for (const auto& batcher : this->batchers_) {
     if (batcher->getStatus() != BatcherStatus::Run) {
-      batcher->start(this);
+      batcher->start(allocators);
     }
   }
   auto thread = worker->spawn(this->batchers_[0]->getOutputQueue());
@@ -226,7 +215,6 @@ void WorkerInfo::unload() {
   }
   auto* worker = this->workers_[id];
   worker->release();
-  worker->deallocate();
   worker->destroy();
 
   if (last_worker) {
@@ -244,44 +232,6 @@ void WorkerInfo::shutdown() {
   }
 }
 
-BufferPtrs WorkerInfo::getInputBuffer() const {
-  BufferPtrs buffer;
-  input_buffer_ptr_->wait_dequeue(buffer);
-  return buffer;
-}
-
-BufferPtrs WorkerInfo::getOutputBuffer() const {
-  BufferPtrs buffer;
-  output_buffer_ptr_->wait_dequeue(buffer);
-  return buffer;
-}
-
-void WorkerInfo::putInputBuffer(BufferPtrs&& buffer) const {
-  this->input_buffer_ptr_->enqueue(std::move(buffer));
-}
-
-void WorkerInfo::putOutputBuffer(BufferPtrs&& buffer) const {
-  this->output_buffer_ptr_->enqueue(std::move(buffer));
-}
-
-bool WorkerInfo::inputSizeValid(size_t size) const {
-  if (size <= this->getBufferNum()) {
-    return true;
-  }
-  if (size <= this->getMaxBufferNum()) {
-    return false;
-  }
-  throw invalid_argument("Too many input tensors for this model");
-}
-
-void WorkerInfo::allocate(size_t request_size) {
-  // TODO(varunsh): can result in deadlock in manager if allocated num !=
-  // request
-  auto allocated =
-    this->workers_.begin()->second->allocate(request_size - this->buffer_num_);
-  this->buffer_num_ += allocated;
-}
-
 ModelMetadata WorkerInfo::getMetadata() const {
   auto* worker_class = workers_.begin()->second;
   return worker_class->getMetadata();
diff --git a/src/amdinfer/core/worker_info.hpp b/src/amdinfer/core/worker_info.hpp
index 7c39a0983..88961abd8 100644
--- a/src/amdinfer/core/worker_info.hpp
+++ b/src/amdinfer/core/worker_info.hpp
@@ -36,6 +36,7 @@ namespace amdinfer {
 class Batcher;
 class ParameterMap;
 class ModelMetadata;
+class MemoryPool;
 namespace workers {
 class Worker;
 }  // namespace workers
@@ -50,7 +51,8 @@ namespace amdinfer {
 class WorkerInfo {
  public:
   /// Construct a new WorkerInfo object
-  WorkerInfo(const std::string& name, ParameterMap* parameters);
+  WorkerInfo(const std::string& name, ParameterMap* parameters,
+             MemoryPool* pool);
   ~WorkerInfo();                           ///> Destroy a WorkerInfo object
   WorkerInfo(WorkerInfo const&) = delete;  ///< Copy constructor
   /// Copy assignment constructor
@@ -76,71 +78,14 @@ class WorkerInfo {
    * @param name
    * @param parameters pointer to parameters. Should not be nullptr
    */
-  void addAndStartWorker(const std::string& name, ParameterMap* parameters);
+  void addAndStartWorker(const std::string& name, ParameterMap* parameters,
+                         MemoryPool* pool);
 
   /// unload one worker from this worker group
   void unload();
   /// unload all workers from this worker group
   void shutdown();
 
-  /**
-   * @brief Get an input buffer from this worker
-   *
-   * @return BufferPtrs
-   */
-  [[nodiscard]] BufferPtrs getInputBuffer() const;
-  /**
-   * @brief Get an output buffer from this worker
-   *
-   * @return BufferPtrs
-   */
-  [[nodiscard]] BufferPtrs getOutputBuffer() const;
-  /**
-   * @brief Return an input buffer to the worker
-   *
-   * @param buffer the buffer to return
-   */
-  void putInputBuffer(BufferPtrs&& buffer) const;
-  /**
-   * @brief Return an output buffer to the worker
-   *
-   * @param buffer the buffer to return
-   */
-  void putOutputBuffer(BufferPtrs&& buffer) const;
-
-  /**
-   * @brief Checks if this worker group supports a particular number of input
-   * tensors.
-   *
-   * @param size
-   * @exception invalid_argument too many input tensors for this worker
-   * @return true if the number of input tensors is acceptable
-   * @return false if more buffers need to be allocated but the number is okay
-   */
-  [[nodiscard]] bool inputSizeValid(size_t size) const;
-
-  /**
-   * @brief Get the number of buffers currently allocated by this worker
-   *
-   * @return auto
-   */
-  [[nodiscard]] auto getBufferNum() const { return this->buffer_num_; }
-
-  /**
-   * @brief Get the maximum number of buffers that this worker may allocate. -1
-   * indicates there is no limit.
-   *
-   * @return auto
-   */
-  [[nodiscard]] auto getMaxBufferNum() const { return this->max_buffer_num_; }
-
-  /**
-   * @brief Allocate enough buffers to fulfill a particular request size
-   *
-   * @param request_size
-   */
-  void allocate(size_t request_size);
-
   /// get the number of workers in the group
   [[nodiscard]] size_t getGroupSize() const;
 
@@ -153,10 +98,6 @@ class WorkerInfo {
   std::map<std::thread::id, std::thread> worker_threads_;
   std::map<std::thread::id, workers::Worker*> workers_;
   std::vector<std::unique_ptr<Batcher>> batchers_;
-  BufferPtrsQueuePtr input_buffer_ptr_;
-  BufferPtrsQueuePtr output_buffer_ptr_;
-  size_t buffer_num_ = 0;
-  size_t max_buffer_num_ = 0;
   size_t batch_size_ = 1;
 
   friend class Manager;
diff --git a/src/amdinfer/observation/tracing.hpp b/src/amdinfer/observation/tracing.hpp
index 118f339f9..a366ee581 100644
--- a/src/amdinfer/observation/tracing.hpp
+++ b/src/amdinfer/observation/tracing.hpp
@@ -90,8 +90,6 @@ class Trace final {
   // std::unique_ptr<opentelemetry::trace::Scope> scope_;
 };
 
-using TracePtr = std::unique_ptr<Trace>;
-
 /// Start a trace with the given name
 TracePtr startTrace(const char* name);
 
diff --git a/src/amdinfer/servers/grpc_server.cpp b/src/amdinfer/servers/grpc_server.cpp
index cc67eba0c..68f5dc9b4 100644
--- a/src/amdinfer/servers/grpc_server.cpp
+++ b/src/amdinfer/servers/grpc_server.cpp
@@ -47,6 +47,7 @@
 #include "amdinfer/core/shared_state.hpp"          // for SharedState
 #include "amdinfer/declarations.hpp"               // for BufferRawPtrs, Infe...
 #include "amdinfer/observation/observer.hpp"       // for Logger, Loggers
+#include "amdinfer/util/containers.hpp"            // for containerProduct
 #include "amdinfer/util/string.hpp"                // for toLower
 #include "amdinfer/util/traits.hpp"                // IWYU pragma: keep
 #include "predict_api.grpc.pb.h"                   // for GRPCInferenceServic...
@@ -479,6 +480,23 @@ class GrpcApiUnary : public Interface {
     return calldata_->getRequest().inputs_size();
   }
 
+  [[nodiscard]] std::vector<size_t> getInputSizes() const override {
+    std::vector<size_t> sizes;
+
+    auto inputs = calldata_->getRequest().inputs();
+
+    for (const auto& tensor : inputs) {
+      const auto& raw_type = tensor.datatype();
+      auto datatype = DataType(raw_type.c_str());
+
+      const auto& shape = tensor.shape();
+      auto size = util::containerProduct(shape);
+
+      sizes.push_back(size * datatype.size());
+    }
+    return sizes;
+  }
+
   void errorHandler(const std::exception& e) override {
     AMDINFER_LOG_INFO(this->getLogger(), e.what());
     calldata_->finish(::grpc::Status(StatusCode::UNKNOWN, e.what()));
diff --git a/src/amdinfer/servers/websocket_server.cpp b/src/amdinfer/servers/websocket_server.cpp
index 815c03f40..78f9f6464 100644
--- a/src/amdinfer/servers/websocket_server.cpp
+++ b/src/amdinfer/servers/websocket_server.cpp
@@ -125,6 +125,33 @@ size_t DrogonWs::getInputSize() {
   return inputs.size();
 }
 
+std::vector<size_t> DrogonWs::getInputSizes() const {
+  std::vector<size_t> sizes;
+
+  if (!this->json_->isMember("inputs")) {
+    throw invalid_argument("No 'inputs' key present in request");
+  }
+  auto inputs = this->json_->get("inputs", Json::arrayValue);
+  if (!inputs.isArray()) {
+    throw invalid_argument("'inputs' is not an array");
+  }
+
+  for (const auto &tensor : inputs) {
+    // using asCString() doesn't work -> the string is empty
+    const auto raw_type = tensor.get("datatype", "UNKNOWN").asString();
+    auto datatype = DataType(raw_type.c_str());
+
+    const auto shape = tensor.get("shape", Json::arrayValue);
+    size_t size = 1;
+    for (const auto &index : shape) {
+      size *= index.asInt64();
+    }
+
+    sizes.push_back(size * datatype.size());
+  }
+  return sizes;
+}
+
 std::shared_ptr<InferenceRequest> DrogonWs::getRequest(
   const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offset,
   const BufferRawPtrs &output_buffers, std::vector<size_t> &output_offset) {
diff --git a/src/amdinfer/servers/websocket_server.hpp b/src/amdinfer/servers/websocket_server.hpp
index 5f4dfee79..7f751846b 100644
--- a/src/amdinfer/servers/websocket_server.hpp
+++ b/src/amdinfer/servers/websocket_server.hpp
@@ -59,6 +59,7 @@ class DrogonWs : public Interface {
     std::vector<size_t> &output_offsets) override;
 
   size_t getInputSize() override;
+  std::vector<size_t> getInputSizes() const override;
   void errorHandler(const std::exception &e) override;
 
  private:
diff --git a/src/amdinfer/workers/CMakeLists.txt b/src/amdinfer/workers/CMakeLists.txt
index 6ae19e101..3c3d4592a 100644
--- a/src/amdinfer/workers/CMakeLists.txt
+++ b/src/amdinfer/workers/CMakeLists.txt
@@ -56,7 +56,6 @@ foreach(worker ${workers})
   set(worker_name worker${worker_pascal})
 
   add_library(${worker_name} SHARED ${file_name}.cpp)
-  target_link_libraries(${worker_name} PRIVATE buffers)
   target_include_directories(${worker_name} PRIVATE ${AMDINFER_INCLUDE_DIRS})
   set_target_options(${worker_name})
 
diff --git a/src/amdinfer/workers/aks.cpp b/src/amdinfer/workers/aks.cpp
index 28bf8873b..0364d890a 100644
--- a/src/amdinfer/workers/aks.cpp
+++ b/src/amdinfer/workers/aks.cpp
@@ -36,21 +36,20 @@
 #include <xir/tensor/tensor.hpp>   // for Tensor
 #include <xir/util/data_type.hpp>  // for create_data_type
 
-#include "amdinfer/batching/batcher.hpp"       // for BatchPtr, Batch, BatchP...
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_TRACING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::Fp32
-#include "amdinfer/core/exceptions.hpp"        // for external_error
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceResponse, Infe...
-#include "amdinfer/declarations.hpp"           // for BufferPtrs, InferenceRe...
-#include "amdinfer/observation/logging.hpp"    // for Logger
-#include "amdinfer/observation/metrics.hpp"    // for Metrics, MetricSummaryIDs
-#include "amdinfer/observation/tracing.hpp"    // for Trace
-#include "amdinfer/util/parse_env.hpp"         // for autoExpandEnvironmentVa...
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/util/timer.hpp"             // for Timer
-#include "amdinfer/workers/worker.hpp"         // for Worker, kNumBufferAuto
+#include "amdinfer/batching/batcher.hpp"     // for BatchPtr, Batch, BatchP...
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_TRACING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::Fp32
+#include "amdinfer/core/exceptions.hpp"      // for external_error
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceResponse, Infe...
+#include "amdinfer/declarations.hpp"         // for BufferPtrs, InferenceRe...
+#include "amdinfer/observation/logging.hpp"  // for Logger
+#include "amdinfer/observation/metrics.hpp"  // for Metrics, MetricSummaryIDs
+#include "amdinfer/observation/tracing.hpp"  // for Trace
+#include "amdinfer/util/parse_env.hpp"       // for autoExpandEnvironmentVa...
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
+#include "amdinfer/workers/worker.hpp"       // for Worker, kNumBufferAuto
 
 namespace AKS {  // NOLINT(readability-identifier-naming)
 class AIGraph;
@@ -68,14 +67,13 @@ class Aks : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   /// the AKS system manager
@@ -88,6 +86,10 @@ std::thread Aks::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&Aks::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> Aks::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void Aks::doInit(ParameterMap* parameters) {
   // arbitrarily set the default batch size to 1
   const int default_batch_size = 1;
@@ -102,17 +104,6 @@ void Aks::doInit(ParameterMap* parameters) {
   this->batch_size_ = batch_size;
 }
 
-size_t Aks::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  VectorBuffer::allocate(this->input_buffers_, kBufferNum,
-                         1 * this->batch_size_, DataType::Fp32);
-  VectorBuffer::allocate(this->output_buffers_, kBufferNum,
-                         1 * this->batch_size_, DataType::Fp32);
-  return buffer_num;
-}
-
 void Aks::doAcquire(ParameterMap* parameters) {
   std::string path{"${AKS_ROOT}/graph_zoo/graph_adder.json"};
   if (parameters->has("aks_graph")) {
@@ -212,7 +203,6 @@ void Aks::doRun(BatchPtrQueue* input_queue) {
 }
 
 void Aks::doRelease() {}
-void Aks::doDeallocate() {}
 void Aks::doDestroy() {}
 
 }  // namespace amdinfer::workers
diff --git a/src/amdinfer/workers/aks_detect.cpp b/src/amdinfer/workers/aks_detect.cpp
index 3aaa229a5..3720ba1c3 100644
--- a/src/amdinfer/workers/aks_detect.cpp
+++ b/src/amdinfer/workers/aks_detect.cpp
@@ -41,22 +41,21 @@
 #include <xir/tensor/tensor.hpp>   // for Tensor
 #include <xir/util/data_type.hpp>  // for create_data_type
 
-#include "amdinfer/batching/batcher.hpp"       // for BatchPtr, Batch, BatchP...
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_TRACING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::Uint32
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceResponse, Infe...
-#include "amdinfer/declarations.hpp"           // for BufferPtrs, InferenceRe...
-#include "amdinfer/observation/logging.hpp"    // for Logger
-#include "amdinfer/observation/metrics.hpp"    // for Metrics, MetricSummaryIDs
-#include "amdinfer/observation/tracing.hpp"    // for Trace
-#include "amdinfer/util/base64.hpp"            // for base64_decode
-#include "amdinfer/util/containers.hpp"        // for containerProduct
-#include "amdinfer/util/parse_env.hpp"         // for autoExpandEnvironmentVa...
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/util/timer.hpp"             // for Timer
-#include "amdinfer/workers/worker.hpp"         // for Worker, kNumBufferAuto
+#include "amdinfer/batching/batcher.hpp"     // for BatchPtr, Batch, BatchP...
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_TRACING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::Uint32
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceResponse, Infe...
+#include "amdinfer/declarations.hpp"         // for BufferPtrs, InferenceRe...
+#include "amdinfer/observation/logging.hpp"  // for Logger
+#include "amdinfer/observation/metrics.hpp"  // for Metrics, MetricSummaryIDs
+#include "amdinfer/observation/tracing.hpp"  // for Trace
+#include "amdinfer/util/base64.hpp"          // for base64_decode
+#include "amdinfer/util/containers.hpp"      // for containerProduct
+#include "amdinfer/util/parse_env.hpp"       // for autoExpandEnvironmentVa...
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
+#include "amdinfer/workers/worker.hpp"       // for Worker, kNumBufferAuto
 
 namespace AKS {  // NOLINT(readability-identifier-naming)
 class AIGraph;
@@ -73,14 +72,13 @@ class AksDetect : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   AKS::SysManagerExt* sys_manager_ = nullptr;
@@ -92,6 +90,10 @@ std::thread AksDetect::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&AksDetect::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> AksDetect::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void AksDetect::doInit(ParameterMap* parameters) {
   this->sys_manager_ = AKS::SysManagerExt::getGlobal();
 
@@ -113,18 +115,6 @@ void AksDetect::doInit(ParameterMap* parameters) {
 constexpr auto kImageWidth = 1920;
 constexpr auto kImageHeight = 1080;
 constexpr auto kImageChannels = 3;
-constexpr auto kImageSize = kImageWidth * kImageHeight * kImageChannels;
-
-size_t AksDetect::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  VectorBuffer::allocate(this->input_buffers_, buffer_num,
-                         kImageSize * this->batch_size_, DataType::Uint8);
-  VectorBuffer::allocate(this->output_buffers_, kBufferNum,
-                         1 * this->batch_size_, DataType::Uint32);
-  return buffer_num;
-}
 
 void AksDetect::doAcquire(ParameterMap* parameters) {
 #ifdef AMDINFER_ENABLE_LOGGING
@@ -277,9 +267,13 @@ void AksDetect::doRun(BatchPtrQueue* input_queue) {
 
         output.setDatatype(DataType::Fp32);
 
-        std::string output_name = outputs[i].getName();
+        std::string output_name;
+        if (i < outputs.size()) {
+          output_name = outputs[i].getName();
+        }
+
         if (output_name.empty()) {
-          output.setName(inputs[i].getName());
+          output.setName(inputs[0].getName());
         } else {
           output.setName(output_name);
         }
@@ -307,12 +301,12 @@ void AksDetect::doRun(BatchPtrQueue* input_queue) {
 #endif
       req->runCallbackOnce(resp);
     }
+    this->returnInputBuffers(std::move(batch));
   }
   AMDINFER_LOG_INFO(logger, "AksDetect ending");
 }
 
 void AksDetect::doRelease() {}
-void AksDetect::doDeallocate() {}
 void AksDetect::doDestroy() {}
 
 }  // namespace amdinfer::workers
diff --git a/src/amdinfer/workers/aks_detect_stream.cpp b/src/amdinfer/workers/aks_detect_stream.cpp
index 4d6544dde..6aa139152 100644
--- a/src/amdinfer/workers/aks_detect_stream.cpp
+++ b/src/amdinfer/workers/aks_detect_stream.cpp
@@ -40,20 +40,19 @@
 #include <xir/tensor/tensor.hpp>   // for Tensor
 #include <xir/util/data_type.hpp>  // for create_data_type
 
-#include "amdinfer/batching/batcher.hpp"       // for BatchPtr, Batch, BatchP...
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_TRACING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::String
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceResponse, Infe...
-#include "amdinfer/declarations.hpp"           // for BufferPtrs, InferenceRe...
-#include "amdinfer/observation/logging.hpp"    // for Logger
-#include "amdinfer/observation/tracing.hpp"    // for Trace
-#include "amdinfer/util/base64.hpp"            // for base64_encode
-#include "amdinfer/util/parse_env.hpp"         // for autoExpandEnvironmentVa...
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/workers/aks_detect.hpp"     // for DetectResponse
-#include "amdinfer/workers/worker.hpp"         // for Worker, kNumBufferAuto
+#include "amdinfer/batching/batcher.hpp"     // for BatchPtr, Batch, BatchP...
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_TRACING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::String
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceResponse, Infe...
+#include "amdinfer/declarations.hpp"         // for BufferPtrs, InferenceRe...
+#include "amdinfer/observation/logging.hpp"  // for Logger
+#include "amdinfer/observation/tracing.hpp"  // for Trace
+#include "amdinfer/util/base64.hpp"          // for base64_encode
+#include "amdinfer/util/parse_env.hpp"       // for autoExpandEnvironmentVa...
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/workers/aks_detect.hpp"   // for DetectResponse
+#include "amdinfer/workers/worker.hpp"       // for Worker, kNumBufferAuto
 
 namespace AKS {  // NOLINT(readability-identifier-naming)
 class AIGraph;
@@ -78,14 +77,13 @@ class AksDetectStream : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   AKS::SysManagerExt* sys_manager_ = nullptr;
@@ -96,6 +94,10 @@ std::thread AksDetectStream::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&AksDetectStream::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> AksDetectStream::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void AksDetectStream::doInit(ParameterMap* parameters) {
   (void)parameters;  // suppress unused variable warning
   constexpr auto kBatchSize = 4;
@@ -109,19 +111,6 @@ void AksDetectStream::doInit(ParameterMap* parameters) {
 constexpr auto kImageWidth = 1920;
 constexpr auto kImageHeight = 1080;
 constexpr auto kImageChannels = 3;
-constexpr auto kImageSize = kImageWidth * kImageHeight * kImageChannels;
-
-size_t AksDetectStream::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  constexpr auto kBufferSize = 128;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  VectorBuffer::allocate(this->input_buffers_, buffer_num, kBufferSize,
-                         DataType::String);
-  VectorBuffer::allocate(this->output_buffers_, buffer_num,
-                         kImageSize * this->batch_size_, DataType::Int8);
-  return buffer_num;
-}
 
 void AksDetectStream::doAcquire(ParameterMap* parameters) {
   std::string path{
@@ -363,7 +352,6 @@ void AksDetectStream::doRun(BatchPtrQueue* input_queue) {
 }
 
 void AksDetectStream::doRelease() {}
-void AksDetectStream::doDeallocate() {}
 void AksDetectStream::doDestroy() {}
 
 }  // namespace workers
diff --git a/src/amdinfer/workers/echo.cpp b/src/amdinfer/workers/echo.cpp
index 99ca05301..72bd9143c 100644
--- a/src/amdinfer/workers/echo.cpp
+++ b/src/amdinfer/workers/echo.cpp
@@ -29,19 +29,18 @@
 #include <utility>    // for move
 #include <vector>     // for vector
 
-#include "amdinfer/batching/hard.hpp"          // for HardBatcher
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_TRACING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::Uint32
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceRequest, Infer...
-#include "amdinfer/declarations.hpp"           // for BufferPtr, InferenceRes...
-#include "amdinfer/observation/logging.hpp"    // for Logger
-#include "amdinfer/observation/metrics.hpp"    // for Metrics
-#include "amdinfer/observation/tracing.hpp"    // for startFollowSpan, SpanPtr
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/util/timer.hpp"             // for Timer
-#include "amdinfer/workers/worker.hpp"         // for Worker
+#include "amdinfer/batching/hard.hpp"        // for HardBatcher
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_TRACING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::Uint32
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceRequest, Infer...
+#include "amdinfer/declarations.hpp"         // for BufferPtr, InferenceRes...
+#include "amdinfer/observation/logging.hpp"  // for Logger
+#include "amdinfer/observation/metrics.hpp"  // for Metrics
+#include "amdinfer/observation/tracing.hpp"  // for startFollowSpan, SpanPtr
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
+#include "amdinfer/workers/worker.hpp"       // for Worker
 
 namespace amdinfer::workers {
 
@@ -55,22 +54,22 @@ class Echo : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   // workers define what batcher implementation should be used for them.
   // if not explicitly defined here, a default value is used from worker.hpp.
   using Worker::makeBatcher;
-  std::vector<std::unique_ptr<Batcher>> makeBatcher(
-    int num, ParameterMap* parameters) override {
-    return this->makeBatcher<HardBatcher>(num, parameters);
+  std::vector<std::unique_ptr<Batcher>> makeBatcher(int num,
+                                                    ParameterMap* parameters,
+                                                    MemoryPool* pool) override {
+    return this->makeBatcher<HardBatcher>(num, parameters, pool);
   };
 };
 
@@ -78,16 +77,13 @@ std::thread Echo::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&Echo::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> Echo::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void Echo::doInit(ParameterMap* parameters) {
-  constexpr auto kMaxBufferNum = 50;
   constexpr auto kBatchSize = 1;
 
-  auto max_buffer_num = kMaxBufferNum;
-  if (parameters->has("max_buffer_num")) {
-    max_buffer_num = parameters->get<int32_t>("max_buffer_num");
-  }
-  this->max_buffer_num_ = max_buffer_num;
-
   auto batch_size = kBatchSize;
   if (parameters->has("batch_size")) {
     batch_size = parameters->get<int32_t>("batch_size");
@@ -95,17 +91,6 @@ void Echo::doInit(ParameterMap* parameters) {
   this->batch_size_ = batch_size;
 }
 
-size_t Echo::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  VectorBuffer::allocate(this->input_buffers_, buffer_num,
-                         1 * this->batch_size_, DataType::Uint32);
-  VectorBuffer::allocate(this->output_buffers_, buffer_num,
-                         1 * this->batch_size_, DataType::Uint32);
-  return buffer_num;
-}
-
 void Echo::doAcquire(ParameterMap* parameters) {
   (void)parameters;  // suppress unused variable warning
 
@@ -163,9 +148,13 @@ void Echo::doRun(BatchPtrQueue* input_queue) {
 
         InferenceResponseOutput output;
         output.setDatatype(DataType::Uint32);
-        std::string output_name = outputs[i].getName();
+        std::string output_name;
+        if (i < outputs.size()) {
+          output_name = outputs[i].getName();
+        }
+
         if (output_name.empty()) {
-          output.setName(inputs[i].getName());
+          output.setName(inputs[0].getName());
         } else {
           output.setName(output_name);
         }
@@ -194,12 +183,12 @@ void Echo::doRun(BatchPtrQueue* input_queue) {
                                             duration);
 #endif
     }
+    this->returnInputBuffers(std::move(batch));
   }
   AMDINFER_LOG_INFO(logger, "Echo ending");
 }
 
 void Echo::doRelease() {}
-void Echo::doDeallocate() {}
 void Echo::doDestroy() {}
 
 }  // namespace amdinfer::workers
diff --git a/src/amdinfer/workers/echo_multi.cpp b/src/amdinfer/workers/echo_multi.cpp
index 1b63bd743..3a6cf1b68 100644
--- a/src/amdinfer/workers/echo_multi.cpp
+++ b/src/amdinfer/workers/echo_multi.cpp
@@ -29,21 +29,20 @@
 #include <utility>  // for move
 #include <vector>   // for vector
 
-#include "amdinfer/batching/hard.hpp"          // for HardBatcher
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_TRACING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::Uint32
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceRequest, Infer...
-#include "amdinfer/declarations.hpp"           // for BufferPtr, InferenceRes...
-#include "amdinfer/observation/logging.hpp"    // for Logger
-#include "amdinfer/observation/metrics.hpp"    // for Metrics
-#include "amdinfer/observation/tracing.hpp"    // for startFollowSpan, SpanPtr
-#include "amdinfer/util/containers.hpp"        // for containerSum
-#include "amdinfer/util/queue.hpp"             // for BufferPtrsQueue
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/util/timer.hpp"             // for Timer
-#include "amdinfer/workers/worker.hpp"         // for Worker
+#include "amdinfer/batching/hard.hpp"        // for HardBatcher
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_TRACING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::Uint32
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceRequest, Infer...
+#include "amdinfer/declarations.hpp"         // for BufferPtr, InferenceRes...
+#include "amdinfer/observation/logging.hpp"  // for Logger
+#include "amdinfer/observation/metrics.hpp"  // for Metrics
+#include "amdinfer/observation/tracing.hpp"  // for startFollowSpan, SpanPtr
+#include "amdinfer/util/containers.hpp"      // for containerSum
+#include "amdinfer/util/queue.hpp"           // for BufferPtrsQueue
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
+#include "amdinfer/workers/worker.hpp"       // for Worker
 
 namespace amdinfer {
 
@@ -63,22 +62,22 @@ class EchoMulti : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   // workers define what batcher implementation should be used for them.
   // if not explicitly defined here, a default value is used from worker.hpp.
   using Worker::makeBatcher;
-  std::vector<std::unique_ptr<Batcher>> makeBatcher(
-    int num, ParameterMap* parameters) override {
-    return this->makeBatcher<HardBatcher>(num, parameters);
+  std::vector<std::unique_ptr<Batcher>> makeBatcher(int num,
+                                                    ParameterMap* parameters,
+                                                    MemoryPool* pool) override {
+    return this->makeBatcher<HardBatcher>(num, parameters, pool);
   };
 };
 
@@ -86,16 +85,13 @@ std::thread EchoMulti::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&EchoMulti::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> EchoMulti::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void EchoMulti::doInit(ParameterMap* parameters) {
-  constexpr auto kMaxBufferNum = 50;
   constexpr auto kBatchSize = 1;
 
-  auto max_buffer_num = kMaxBufferNum;
-  if (parameters->has("max_buffer_num")) {
-    max_buffer_num = parameters->get<int32_t>("max_buffer_num");
-  }
-  this->max_buffer_num_ = max_buffer_num;
-
   auto batch_size = kBatchSize;
   if (parameters->has("batch_size")) {
     batch_size = parameters->get<int32_t>("batch_size");
@@ -103,30 +99,6 @@ void EchoMulti::doInit(ParameterMap* parameters) {
   this->batch_size_ = batch_size;
 }
 
-size_t EchoMulti::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-
-  for (size_t i = 0; i < buffer_num; ++i) {
-    BufferPtrs vec;
-    for (auto j = 0; j < kInputTensors; ++j) {
-      vec.emplace_back(std::make_unique<VectorBuffer>(
-        kInputLengths.at(j) * this->batch_size_, DataType::Uint32));
-    }
-    this->input_buffers_->enqueue(std::move(vec));
-  }
-  for (size_t i = 0; i < buffer_num; i++) {
-    BufferPtrs vec;
-    for (auto j = 0; j < kOutputTensors; ++j) {
-      vec.emplace_back(std::make_unique<VectorBuffer>(
-        kOutputLengths.at(j) * this->batch_size_, DataType::Uint32));
-    }
-    this->output_buffers_->enqueue(std::move(vec));
-  }
-  return buffer_num;
-}
-
 void EchoMulti::doAcquire([[maybe_unused]] ParameterMap* parameters) {
   for (auto i = 0; i < kInputTensors; ++i) {
     this->metadata_.addInputTensor(
@@ -193,7 +165,11 @@ void EchoMulti::doRun(BatchPtrQueue* input_queue) {
 
         InferenceResponseOutput output;
         output.setDatatype(DataType::Uint32);
-        std::string output_name = outputs[i].getName();
+        std::string output_name;
+        if (static_cast<size_t>(i) < outputs.size()) {
+          output_name = outputs[i].getName();
+        }
+
         if (output_name.empty()) {
           output.setName(inputs[0].getName());
         } else {
@@ -226,12 +202,12 @@ void EchoMulti::doRun(BatchPtrQueue* input_queue) {
                                             duration);
 #endif
     }
+    this->returnInputBuffers(std::move(batch));
   }
   AMDINFER_LOG_INFO(logger, "EchoMulti ending");
 }
 
 void EchoMulti::doRelease() {}
-void EchoMulti::doDeallocate() {}
 void EchoMulti::doDestroy() {}
 
 }  // namespace workers
diff --git a/src/amdinfer/workers/invert_image.cpp b/src/amdinfer/workers/invert_image.cpp
index 3a13e3db0..9c76d271b 100644
--- a/src/amdinfer/workers/invert_image.cpp
+++ b/src/amdinfer/workers/invert_image.cpp
@@ -32,21 +32,20 @@
 #include <utility>                // for move
 #include <vector>                 // for vector
 
-#include "amdinfer/batching/batcher.hpp"       // for Batch, BatchPtrQueue
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_TRACING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::Uint8
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceRequest, Infer...
-#include "amdinfer/declarations.hpp"           // for BufferPtr, InferenceRes...
-#include "amdinfer/observation/logging.hpp"    // for Logger
-#include "amdinfer/observation/metrics.hpp"    // for Metrics
-#include "amdinfer/observation/tracing.hpp"    // for startFollowSpan, SpanPtr
-#include "amdinfer/util/base64.hpp"            // for base64_decode, base64_e...
-#include "amdinfer/util/containers.hpp"        // for containerProduct
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/util/timer.hpp"             // for Timer
-#include "amdinfer/workers/worker.hpp"         // for Worker
+#include "amdinfer/batching/batcher.hpp"     // for Batch, BatchPtrQueue
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_TRACING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::Uint8
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceRequest, Infer...
+#include "amdinfer/declarations.hpp"         // for BufferPtr, InferenceRes...
+#include "amdinfer/observation/logging.hpp"  // for Logger
+#include "amdinfer/observation/metrics.hpp"  // for Metrics
+#include "amdinfer/observation/tracing.hpp"  // for startFollowSpan, SpanPtr
+#include "amdinfer/util/base64.hpp"          // for base64_decode, base64_e...
+#include "amdinfer/util/containers.hpp"      // for containerProduct
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
+#include "amdinfer/workers/worker.hpp"       // for Worker
 
 namespace {
 
@@ -85,14 +84,13 @@ class InvertImage : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 };
 
@@ -100,16 +98,13 @@ std::thread InvertImage::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&InvertImage::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> InvertImage::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void InvertImage::doInit(ParameterMap* parameters) {
-  constexpr auto kMaxBufferNum = 50;
   constexpr auto kBatchSize = 1;
 
-  auto max_buffer_num = kMaxBufferNum;
-  if (parameters->has("max_buffer_num")) {
-    max_buffer_num = parameters->get<int32_t>("max_buffer_num");
-  }
-  this->max_buffer_num_ = max_buffer_num;
-
   auto batch_size = kBatchSize;
   if (parameters->has("batch_size")) {
     batch_size = parameters->get<int32_t>("batch_size");
@@ -122,19 +117,6 @@ const auto kMaxImageHeight = 1080;
 const auto kMaxImageWidth = 1920;
 const auto kMaxImageChannels = 3;
 
-size_t InvertImage::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  constexpr auto kBufferSize =
-    kMaxImageWidth * kMaxImageHeight * kMaxImageChannels;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  VectorBuffer::allocate(this->input_buffers_, buffer_num,
-                         kBufferSize * this->batch_size_, DataType::Uint8);
-  VectorBuffer::allocate(this->output_buffers_, buffer_num,
-                         kBufferSize * this->batch_size_, DataType::Uint8);
-  return buffer_num;
-}
-
 void InvertImage::doAcquire(ParameterMap* parameters) {
   (void)parameters;  // suppress unused variable warning
 
@@ -152,6 +134,9 @@ void InvertImage::doRun(BatchPtrQueue* input_queue) {
   const auto& logger = this->getLogger();
 #endif
 
+  std::vector<std::byte> output;
+  output.resize(kMaxImageHeight * kMaxImageWidth * kMaxImageChannels);
+
   while (true) {
     BatchPtr batch;
     input_queue->wait_dequeue(batch);
@@ -173,7 +158,7 @@ void InvertImage::doRun(BatchPtrQueue* input_queue) {
       auto outputs = req->getOutputs();
       for (unsigned int i = 0; i < inputs.size(); i++) {
         auto* input_buffer = inputs[i].getData();
-        auto* output_buffer = outputs[i].getData();
+        auto* output_buffer = output.data();
 
         auto input_shape = inputs[i].getShape();
 
@@ -192,7 +177,7 @@ void InvertImage::doRun(BatchPtrQueue* input_queue) {
             invert<uint8_t*, true>(input_buffer, output_buffer, input_size);
           }
 
-          auto* output_data = static_cast<uint8_t*>(output_buffer);
+          auto* output_data = reinterpret_cast<uint8_t*>(output_buffer);
 
           std::vector<std::byte> buffer;
           buffer.resize(input_size);
@@ -233,9 +218,13 @@ void InvertImage::doRun(BatchPtrQueue* input_queue) {
 
         // if our output is explicitly named, use that name in response, or use
         // the input tensor's name if it's not defined.
-        std::string output_name = outputs[i].getName();
+        std::string output_name;
+        if (i < outputs.size()) {
+          output_name = outputs[i].getName();
+        }
+
         if (output_name.empty()) {
-          output.setName(inputs[i].getName());
+          output.setName(inputs[0].getName());
         } else {
           output.setName(output_name);
         }
@@ -255,12 +244,12 @@ void InvertImage::doRun(BatchPtrQueue* input_queue) {
 #endif
       req->runCallbackOnce(resp);
     }
+    this->returnInputBuffers(std::move(batch));
   }
   AMDINFER_LOG_INFO(logger, "InvertImage ending");
 }
 
 void InvertImage::doRelease() {}
-void InvertImage::doDeallocate() {}
 void InvertImage::doDestroy() {}
 
 }  // namespace amdinfer::workers
diff --git a/src/amdinfer/workers/invert_video.cpp b/src/amdinfer/workers/invert_video.cpp
index 879025429..5eb841500 100644
--- a/src/amdinfer/workers/invert_video.cpp
+++ b/src/amdinfer/workers/invert_video.cpp
@@ -28,18 +28,17 @@
 #include <thread>                 // for thread
 #include <vector>                 // for vector
 
-#include "amdinfer/batching/batcher.hpp"       // for Batch, BatchPtrQueue
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_TRACING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::String
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceResponse, Infe...
-#include "amdinfer/declarations.hpp"           // for BufferPtr, InferenceRes...
-#include "amdinfer/observation/logging.hpp"    // for Logger
-#include "amdinfer/observation/tracing.hpp"    // for startFollowSpan, SpanPtr
-#include "amdinfer/util/base64.hpp"            // for base64_encode
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/workers/worker.hpp"         // for Worker
+#include "amdinfer/batching/batcher.hpp"     // for Batch, BatchPtrQueue
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_TRACING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::String
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceResponse, Infe...
+#include "amdinfer/declarations.hpp"         // for BufferPtr, InferenceRes...
+#include "amdinfer/observation/logging.hpp"  // for Logger
+#include "amdinfer/observation/tracing.hpp"  // for startFollowSpan, SpanPtr
+#include "amdinfer/util/base64.hpp"          // for base64_encode
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/workers/worker.hpp"       // for Worker
 
 namespace amdinfer {
 
@@ -60,14 +59,13 @@ class InvertVideo : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 };
 
@@ -75,15 +73,12 @@ std::thread InvertVideo::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&InvertVideo::run, this, input_queue);
 }
 
-void InvertVideo::doInit(ParameterMap* parameters) {
-  constexpr auto kMaxBufferNum = 50;
-  constexpr auto kBatchSize = 1;
+std::vector<MemoryAllocators> InvertVideo::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
 
-  auto max_buffer_num = kMaxBufferNum;
-  if (parameters->has("max_buffer_num")) {
-    max_buffer_num = parameters->get<int32_t>("max_buffer_num");
-  }
-  this->max_buffer_num_ = max_buffer_num;
+void InvertVideo::doInit([[maybe_unused]] ParameterMap* parameters) {
+  constexpr auto kBatchSize = 1;
 
   this->batch_size_ = kBatchSize;
 }
@@ -96,19 +91,6 @@ const auto kMaxImageChannels = 3;
 // arbitrarily choose max URL length for the video
 const auto kMaxUrlLength = 128;
 
-size_t InvertVideo::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  constexpr auto kBufferSize = kMaxUrlLength;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  VectorBuffer::allocate(this->input_buffers_, buffer_num, kBufferSize,
-                         DataType::String);
-  VectorBuffer::allocate(this->output_buffers_, buffer_num,
-                         kMaxImageWidth * kMaxImageHeight * kMaxImageChannels,
-                         DataType::Int8);
-  return buffer_num;
-}
-
 void InvertVideo::doAcquire(ParameterMap* parameters) {
   (void)parameters;  // suppress unused variable warning
 
@@ -209,7 +191,6 @@ void InvertVideo::doRun(BatchPtrQueue* input_queue) {
 }
 
 void InvertVideo::doRelease() {}
-void InvertVideo::doDeallocate() {}
 void InvertVideo::doDestroy() {}
 
 }  // namespace workers
diff --git a/src/amdinfer/workers/migraphx.cpp b/src/amdinfer/workers/migraphx.cpp
index 834ac5805..ea4c6f86b 100644
--- a/src/amdinfer/workers/migraphx.cpp
+++ b/src/amdinfer/workers/migraphx.cpp
@@ -35,21 +35,20 @@
 #include <utility>                // for move
 #include <vector>                 // for vector
 
-#include "amdinfer/batching/hard.hpp"          // for BatchPtr, Batch, Batch...
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_LOGGING
-#include "amdinfer/core/data_types.hpp"        // for DataType, operator<<
-#include "amdinfer/core/exceptions.hpp"        // for invalid_argument, runt...
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceRequest, Infe...
-#include "amdinfer/declarations.hpp"           // for InferenceResponseOutput
-#include "amdinfer/observation/logging.hpp"    // for AMDINFER_LOG_INFO, AMD...
-#include "amdinfer/observation/metrics.hpp"    // for Metrics, MetricCounterIDs
-#include "amdinfer/util/containers.hpp"        // for containerProduct
-#include "amdinfer/util/queue.hpp"             // for BufferPtrsQueue
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/util/timer.hpp"             // for Timer
-#include "amdinfer/workers/worker.hpp"         // for Worker, kNumBufferAuto
+#include "amdinfer/batching/hard.hpp"        // for BatchPtr, Batch, Batch...
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_LOGGING
+#include "amdinfer/core/data_types.hpp"      // for DataType, operator<<
+#include "amdinfer/core/exceptions.hpp"      // for invalid_argument, runt...
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceRequest, Infe...
+#include "amdinfer/declarations.hpp"         // for InferenceResponseOutput
+#include "amdinfer/observation/logging.hpp"  // for AMDINFER_LOG_INFO, AMD...
+#include "amdinfer/observation/metrics.hpp"  // for Metrics, MetricCounterIDs
+#include "amdinfer/util/containers.hpp"      // for containerProduct
+#include "amdinfer/util/queue.hpp"           // for BufferPtrsQueue
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
+#include "amdinfer/workers/worker.hpp"       // for Worker, kNumBufferAuto
 
 namespace amdinfer::workers {
 
@@ -62,14 +61,13 @@ class MIGraphXWorker : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   // the model file to be loaded.  Supported types are *.onnx and *.mxr
@@ -92,6 +90,10 @@ std::thread MIGraphXWorker::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&MIGraphXWorker::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> MIGraphXWorker::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 // Enum-to-enum conversion to let us read data type from migraphx model.
 // The definitions are taken from the MIGraphX macro
 // MIGRAPHX_SHAPE_VISIT_TYPES
@@ -281,44 +283,7 @@ void MIGraphXWorker::doInit(ParameterMap* parameters) {
   auto length = sh.lengths();
   migraphx::api::shapes output_shapes = prog_.get_output_shapes();
   this->batch_size_ = length[0];
-}
-
-/**
- * @brief doAllocate() allocates a queue of buffers for input/output.
- * A functioning doAllocate() is required or else the worker will
- * simply fail to respond to post requests.
- * doAllocate() may be called multiple times by the engine
- *  to allocate more buffer space if necessary.
- *
- * @param num the number of buffers to add
- * @return size_t the number of buffers added
- */
-size_t MIGraphXWorker::doAllocate(size_t num) {
-#ifdef AMDINFER_ENABLE_LOGGING
-  const auto& logger = this->getLogger();
-#endif
-  AMDINFER_LOG_INFO(logger, "MIGraphXWorker::doAllocate");
-  //
-  // Allocate
-  //
-  constexpr auto kBufferNum = 3U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  // Allocate enough to hold buffer_num batches' worth of input sets.
-  // Extra batches allow server to hold more requests at one time
-  // todo:  this try/catch was observed to just get stuck when batch size
-  // is too big (approx. 56 for Yolov4 model); how to catch the error?
-
-  // Calculate the total number of bytes required for all inputs
-
-  BufferPtrs buffer_vec;
 
-  migraphx::program_parameter_shapes input_shapes =
-    this->prog_.get_parameter_shapes();
-
-  // Work out the max. size of any input buffer, in bytes.  We'll allocate all
-  // of them the same size in case a request puts them in mixed-up order.
-  size_t max_buffer(0);
   for (const auto* aname : input_shapes.names()) {
     migraphx::shape ashape = input_shapes[aname];
     auto llen = ashape.lengths();
@@ -326,47 +291,7 @@ size_t MIGraphXWorker::doAllocate(size_t num) {
     auto asize = ashape.bytes();
     // size of a single request input (divide by batch size)
     input_sizes_[aname] = asize / *(ashape.lengths().begin());
-    max_buffer = std::max(max_buffer, asize);
-  }
-
-  // Now, allocate the input and output buffers.
-
-  try {
-    for (const auto* aname : input_shapes.names()) {
-      auto ashape = input_shapes[aname];
-      auto llen = ashape.lengths();
-
-      // todo: test whether VectorBuffer::allocate() does this in the right
-      // order for multiple (kBufferNum) sets of buffers. It wasn't designed to
-      // be called in a loop like this.  Using 1 in place of kBufferNum
-
-      buffer_vec.emplace_back(
-        std::make_unique<VectorBuffer>(max_buffer, DataType::Uint8));
-    }
-    this->input_buffers_->enqueue(std::move(buffer_vec));
-
-    // Calculate max. output buffer size
-    size_t out_buffer_size{0};
-    migraphx::shapes output_shapes = this->prog_.get_output_shapes();
-    for (const auto& ash : output_shapes) {
-      out_buffer_size = std::max(out_buffer_size, ash.bytes());
-    }
-
-    // Output buffers aren't used by the engine at time of writing this,
-    // but allocate them anyway. (Use number of outputs for kBufferNum)
-    VectorBuffer::allocate(this->output_buffers_, output_shapes.size(),
-                           out_buffer_size, amdinfer::DataType::Int8);
-  } catch (...) {
-    AMDINFER_LOG_ERROR(
-      logger,
-      std::string("MIGraphXWorker couldn't allocate buffer (batch size ") +
-        std::to_string(batch_size_) + ")");
-    throw runtime_error{"MIGraphXWorker couldn't allocate buffer"};
   }
-  AMDINFER_LOG_INFO(logger, std::string("MIGraphXWorker::doAllocate() added ") +
-                              std::to_string(buffer_num) + " buffers");
-
-  return buffer_num;
 }
 
 void MIGraphXWorker::doAcquire(ParameterMap* parameters) { (void)parameters; }
@@ -625,6 +550,7 @@ void MIGraphXWorker::doRun(BatchPtrQueue* input_queue) {
     }
 
     timer.add("batch_stop");
+    this->returnInputBuffers(std::move(batch));
     auto duration = timer.count<std::micro>("batch_start", "batch_stop");
     AMDINFER_LOG_INFO(
       logger, std::string("Finished migraphx batch processing; batch size: ") +
@@ -635,7 +561,6 @@ void MIGraphXWorker::doRun(BatchPtrQueue* input_queue) {
 }
 
 void MIGraphXWorker::doRelease() {}
-void MIGraphXWorker::doDeallocate() {}
 void MIGraphXWorker::doDestroy() {}
 
 }  // namespace amdinfer::workers
diff --git a/src/amdinfer/workers/pt_zendnn.cpp b/src/amdinfer/workers/pt_zendnn.cpp
index ee070bd6f..800a86708 100644
--- a/src/amdinfer/workers/pt_zendnn.cpp
+++ b/src/amdinfer/workers/pt_zendnn.cpp
@@ -30,22 +30,21 @@
 #include <utility>     // for move
 #include <vector>      // for vector
 
-#include "amdinfer/batching/hard.hpp"          // for Batch, BatchPtrQueue
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_LOGGING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::FP32
-#include "amdinfer/core/exceptions.hpp"        // for external_error, file_no...
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceResponse, Requ...
-#include "amdinfer/declarations.hpp"           // for InferenceResponseOutput
-#include "amdinfer/observation/logging.hpp"    // for Logger, AMDINFER_LOG_INFO
-#include "amdinfer/observation/metrics.hpp"    // for Metrics, MetricCounterIDs
-#include "amdinfer/observation/tracing.hpp"    // for Trace
-#include "amdinfer/util/containers.hpp"        // for containerProduct
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/util/timer.hpp"             // for Timer
-#include "amdinfer/workers/worker.hpp"         // for Worker, kNumBufferAuto
-#include "torch/script.h"                      // for IValue, Tensor, Device
+#include "amdinfer/batching/hard.hpp"        // for Batch, BatchPtrQueue
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_LOGGING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::FP32
+#include "amdinfer/core/exceptions.hpp"      // for external_error, file_no...
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceResponse, Requ...
+#include "amdinfer/declarations.hpp"         // for InferenceResponseOutput
+#include "amdinfer/observation/logging.hpp"  // for Logger, AMDINFER_LOG_INFO
+#include "amdinfer/observation/metrics.hpp"  // for Metrics, MetricCounterIDs
+#include "amdinfer/observation/tracing.hpp"  // for Trace
+#include "amdinfer/util/containers.hpp"      // for containerProduct
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
+#include "amdinfer/workers/worker.hpp"       // for Worker, kNumBufferAuto
+#include "torch/script.h"                    // for IValue, Tensor, Device
 
 namespace fs = std::filesystem;
 
@@ -65,14 +64,13 @@ class PtZendnn : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   // workers define what batcher implementation should be used for them.
@@ -100,16 +98,13 @@ std::thread PtZendnn::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&PtZendnn::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> PtZendnn::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void PtZendnn::doInit(ParameterMap* parameters) {
-  constexpr auto kMaxBufferNum = 64;
   constexpr auto kBatchSize = 1;
 
-  auto max_buffer_num = kMaxBufferNum;
-  if (parameters->has("max_buffer_num")) {
-    max_buffer_num = parameters->get<int32_t>("max_buffer_num");
-  }
-  this->max_buffer_num_ = max_buffer_num;
-
   auto batch_size = kBatchSize;
   if (parameters->has("batch_size")) {
     batch_size = parameters->get<int32_t>("batch_size");
@@ -132,18 +127,6 @@ void PtZendnn::doInit(ParameterMap* parameters) {
   }
 }
 
-size_t PtZendnn::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-
-  VectorBuffer::allocate(this->input_buffers_, buffer_num,
-                         image_size_ * this->batch_size_, input_dt_);
-  VectorBuffer::allocate(this->output_buffers_, buffer_num,
-                         output_classes_ * this->batch_size_, DataType::FP32);
-  return buffer_num;
-}
-
 void PtZendnn::doAcquire(ParameterMap* parameters) {
 #ifdef AMDINFER_ENABLE_LOGGING
   const auto& logger = this->getLogger();
@@ -304,9 +287,13 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
                response_size * sizeof(float));
         output.setData(std::move(buffer));
 
-        std::string output_name = outputs[i].getName();
+        std::string output_name;
+        if (i < outputs.size()) {
+          output_name = outputs[i].getName();
+        }
+
         if (output_name.empty()) {
-          output.setName(inputs[i].getName());
+          output.setName(inputs[0].getName());
         } else {
           output.setName(output_name);
         }
@@ -334,12 +321,12 @@ void PtZendnn::doRun(BatchPtrQueue* input_queue) {
                                             duration);
 #endif
     }
+    this->returnInputBuffers(std::move(batch));
   }
   AMDINFER_LOG_INFO(logger, "PtZendnn ending");
 }
 
 void PtZendnn::doRelease() {}
-void PtZendnn::doDeallocate() {}
 void PtZendnn::doDestroy() {}
 
 }  // namespace amdinfer::workers
diff --git a/src/amdinfer/workers/resnet50.cpp b/src/amdinfer/workers/resnet50.cpp
index ee09deda9..b2e14ebce 100644
--- a/src/amdinfer/workers/resnet50.cpp
+++ b/src/amdinfer/workers/resnet50.cpp
@@ -38,22 +38,21 @@
 #include <xir/tensor/tensor.hpp>   // for Tensor
 #include <xir/util/data_type.hpp>  // for create_data_type
 
-#include "amdinfer/batching/batcher.hpp"       // for BatchPtr, Batch, BatchP...
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_TRACING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::Uint32
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceResponse, Infe...
-#include "amdinfer/declarations.hpp"           // for BufferPtrs, InferenceRe...
-#include "amdinfer/observation/logging.hpp"    // for Logger
-#include "amdinfer/observation/metrics.hpp"    // for Metrics, MetricSummaryIDs
-#include "amdinfer/observation/tracing.hpp"    // for Trace
-#include "amdinfer/util/base64.hpp"            // for base64_decode
-#include "amdinfer/util/containers.hpp"        // for containerProduct
-#include "amdinfer/util/parse_env.hpp"         // for autoExpandEnvironmentVa...
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/util/timer.hpp"             // for Timer
-#include "amdinfer/workers/worker.hpp"         // for Worker, kNumBufferAuto
+#include "amdinfer/batching/batcher.hpp"     // for BatchPtr, Batch, BatchP...
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_TRACING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::Uint32
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceResponse, Infe...
+#include "amdinfer/declarations.hpp"         // for BufferPtrs, InferenceRe...
+#include "amdinfer/observation/logging.hpp"  // for Logger
+#include "amdinfer/observation/metrics.hpp"  // for Metrics, MetricSummaryIDs
+#include "amdinfer/observation/tracing.hpp"  // for Trace
+#include "amdinfer/util/base64.hpp"          // for base64_decode
+#include "amdinfer/util/containers.hpp"      // for containerProduct
+#include "amdinfer/util/parse_env.hpp"       // for autoExpandEnvironmentVa...
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
+#include "amdinfer/workers/worker.hpp"       // for Worker, kNumBufferAuto
 
 namespace AKS {  // NOLINT(readability-identifier-naming)
 class AIGraph;
@@ -70,14 +69,13 @@ class ResNet50 : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   AKS::SysManagerExt* sys_manager_ = nullptr;
@@ -89,6 +87,10 @@ std::thread ResNet50::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&ResNet50::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> ResNet50::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void ResNet50::doInit(ParameterMap* parameters) {
   // DPUCADF8H uses batch size of 4 by default
   const auto default_batch_size = 4;
@@ -111,18 +113,6 @@ void ResNet50::doInit(ParameterMap* parameters) {
 constexpr auto kImageWidth = 1920;
 constexpr auto kImageHeight = 1080;
 constexpr auto kImageChannels = 3;
-constexpr auto kImageSize = kImageWidth * kImageHeight * kImageChannels;
-
-size_t ResNet50::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  VectorBuffer::allocate(this->input_buffers_, buffer_num,
-                         kImageSize * this->batch_size_, DataType::Uint8);
-  VectorBuffer::allocate(this->output_buffers_, kBufferNum,
-                         1 * this->batch_size_, DataType::Uint32);
-  return buffer_num;
-}
 
 void ResNet50::doAcquire(ParameterMap* parameters) {
   std::string path{
@@ -267,9 +257,13 @@ void ResNet50::doRun(BatchPtrQueue* input_queue) {
                response_size * sizeof(int));
         output.setData(std::move(buffer));
 
-        std::string output_name = outputs[i].getName();
+        std::string output_name;
+        if (i < outputs.size()) {
+          output_name = outputs[i].getName();
+        }
+
         if (output_name.empty()) {
-          output.setName(inputs[i].getName());
+          output.setName(inputs[0].getName());
         } else {
           output.setName(output_name);
         }
@@ -293,12 +287,12 @@ void ResNet50::doRun(BatchPtrQueue* input_queue) {
 #endif
       req->runCallbackOnce(resp);
     }
+    this->returnInputBuffers(std::move(batch));
   }
   AMDINFER_LOG_INFO(logger, "ResNet50 ending");
 }
 
 void ResNet50::doRelease() {}
-void ResNet50::doDeallocate() {}
 void ResNet50::doDestroy() {}
 
 }  // namespace amdinfer::workers
diff --git a/src/amdinfer/workers/resnet50_stream.cpp b/src/amdinfer/workers/resnet50_stream.cpp
index 23051e285..c27011e52 100644
--- a/src/amdinfer/workers/resnet50_stream.cpp
+++ b/src/amdinfer/workers/resnet50_stream.cpp
@@ -41,18 +41,17 @@
 #include <xir/tensor/tensor.hpp>   // for Tensor
 #include <xir/util/data_type.hpp>  // for create_data_type
 
-#include "amdinfer/batching/batcher.hpp"       // for BatchPtr, BatchPtrQueue
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_LOGGING
-#include "amdinfer/core/data_types.hpp"        // for DataType, DataType::String
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceResponse, Infe...
-#include "amdinfer/declarations.hpp"           // for BufferPtrs, InferenceRe...
-#include "amdinfer/observation/logging.hpp"    // for Logger
-#include "amdinfer/util/base64.hpp"            // for base64_encode
-#include "amdinfer/util/parse_env.hpp"         // for autoExpandEnvironmentVa...
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/workers/worker.hpp"         // for Worker, kNumBufferAuto
+#include "amdinfer/batching/batcher.hpp"     // for BatchPtr, BatchPtrQueue
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_LOGGING
+#include "amdinfer/core/data_types.hpp"      // for DataType, DataType::String
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceResponse, Infe...
+#include "amdinfer/declarations.hpp"         // for BufferPtrs, InferenceRe...
+#include "amdinfer/observation/logging.hpp"  // for Logger
+#include "amdinfer/util/base64.hpp"          // for base64_encode
+#include "amdinfer/util/parse_env.hpp"       // for autoExpandEnvironmentVa...
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/workers/worker.hpp"       // for Worker, kNumBufferAuto
 
 namespace AKS {  // NOLINT(readability-identifier-naming)
 class AIGraph;
@@ -79,14 +78,13 @@ class ResNet50Stream : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   AKS::SysManagerExt* sys_manager_ = nullptr;
@@ -98,6 +96,10 @@ std::thread ResNet50Stream::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&ResNet50Stream::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> ResNet50Stream::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void ResNet50Stream::doInit(ParameterMap* parameters) {
   constexpr auto kBatchSize = 4;
   (void)parameters;  // suppress unused variable warning
@@ -124,17 +126,6 @@ const std::string kBoxHeightStr{"10"};
 /// number of categories returned for the image
 constexpr auto kResnetClassifications = 5;
 
-size_t ResNet50Stream::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-  VectorBuffer::allocate(this->input_buffers_, buffer_num,
-                         kImageSize * this->batch_size_, DataType::Uint8);
-  VectorBuffer::allocate(this->output_buffers_, buffer_num,
-                         kImageSize * this->batch_size_, DataType::Uint8);
-  return buffer_num;
-}
-
 void ResNet50Stream::doAcquire(ParameterMap* parameters) {
   std::string path{
     "${AKS_ROOT}/graph_zoo/graph_tf_resnet_v1_50_u200_u250_amdinfer.json"};
@@ -327,7 +318,6 @@ void ResNet50Stream::doRun(BatchPtrQueue* input_queue) {
 }
 
 void ResNet50Stream::doRelease() {}
-void ResNet50Stream::doDeallocate() {}
 void ResNet50Stream::doDestroy() {}
 
 }  // namespace workers
diff --git a/src/amdinfer/workers/tf_zendnn.cpp b/src/amdinfer/workers/tf_zendnn.cpp
index 8cff41530..68f1d1153 100644
--- a/src/amdinfer/workers/tf_zendnn.cpp
+++ b/src/amdinfer/workers/tf_zendnn.cpp
@@ -43,20 +43,19 @@
 #include <utility>    // for pair, move
 #include <vector>     // for vector
 
-#include "amdinfer/batching/hard.hpp"          // for Batch, BatchP...
-#include "amdinfer/buffers/vector_buffer.hpp"  // for VectorBuffer
-#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABL...
-#include "amdinfer/core/data_types.hpp"        // for DataType, Dat...
-#include "amdinfer/core/exceptions.hpp"        // for external_error
-#include "amdinfer/core/parameters.hpp"        // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"       // for InferenceResp...
-#include "amdinfer/declarations.hpp"           // for InferenceResp...
-#include "amdinfer/observation/logging.hpp"    // for Logger, PROTE...
-#include "amdinfer/observation/metrics.hpp"    // for Metrics, Metr...
-#include "amdinfer/observation/tracing.hpp"    // for Trace
-#include "amdinfer/util/thread.hpp"            // for setThreadName
-#include "amdinfer/util/timer.hpp"             // for Timer
-#include "amdinfer/workers/worker.hpp"         // for Worker, kNumB...
+#include "amdinfer/batching/hard.hpp"        // for Batch, BatchP...
+#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABL...
+#include "amdinfer/core/data_types.hpp"      // for DataType, Dat...
+#include "amdinfer/core/exceptions.hpp"      // for external_error
+#include "amdinfer/core/parameters.hpp"      // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"     // for InferenceResp...
+#include "amdinfer/declarations.hpp"         // for InferenceResp...
+#include "amdinfer/observation/logging.hpp"  // for Logger, PROTE...
+#include "amdinfer/observation/metrics.hpp"  // for Metrics, Metr...
+#include "amdinfer/observation/tracing.hpp"  // for Trace
+#include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
+#include "amdinfer/workers/worker.hpp"       // for Worker, kNumB...
 
 namespace tf = ::tensorflow;
 
@@ -76,14 +75,13 @@ class TfZendnn : public Worker {
  public:
   using Worker::Worker;
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   // TF session and graphs
@@ -108,14 +106,12 @@ std::thread TfZendnn::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&TfZendnn::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> TfZendnn::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 void TfZendnn::doInit(ParameterMap* parameters) {
   const auto default_batch_size = 1;
-  const int default_max_buffer_num = 64;
-
-  max_buffer_num_ = default_max_buffer_num;
-  if (parameters->has("max_buffer_num")) {
-    max_buffer_num_ = parameters->get<int32_t>("max_buffer_num");
-  }
 
   auto batch_size = default_batch_size;
   if (parameters->has("batch_size")) {
@@ -153,18 +149,6 @@ void TfZendnn::doInit(ParameterMap* parameters) {
 #endif
 }
 
-size_t TfZendnn::doAllocate(size_t num) {
-  const auto default_buffer_num = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? default_buffer_num : num;
-
-  VectorBuffer::allocate(this->input_buffers_, buffer_num,
-                         image_size_ * this->batch_size_, input_dt_);
-  VectorBuffer::allocate(this->output_buffers_, buffer_num,
-                         output_classes_ * this->batch_size_, DataType::Fp32);
-  return buffer_num;
-}
-
 void TfZendnn::doAcquire(ParameterMap* parameters) {
 #ifdef AMDINFER_ENABLE_LOGGING
   const auto& logger = this->getLogger();
@@ -331,9 +315,13 @@ void TfZendnn::doRun(BatchPtrQueue* input_queue) {
                response_size * sizeof(float));
         output.setData(std::move(buffer));
 
-        std::string output_name = outputs[i].getName();
+        std::string output_name;
+        if (i < outputs.size()) {
+          output_name = outputs[i].getName();
+        }
+
         if (output_name.empty()) {
-          output.setName(inputs[i].getName());
+          output.setName(inputs[0].getName());
         } else {
           output.setName(output_name);
         }
@@ -362,6 +350,7 @@ void TfZendnn::doRun(BatchPtrQueue* input_queue) {
                                             duration);
 #endif
     }
+    this->returnInputBuffers(std::move(batch));
   }
   AMDINFER_LOG_INFO(logger, "TfZendnn ending");
 }
@@ -370,7 +359,6 @@ void TfZendnn::doRelease() {
   auto retval = this->session_->Close();
   assert(retval.ok());
 }
-void TfZendnn::doDeallocate() {}
 void TfZendnn::doDestroy() {}
 
 void* openLibrary(const char* library, int dlopen_flags) {
diff --git a/src/amdinfer/workers/worker.hpp b/src/amdinfer/workers/worker.hpp
index 042ff0fb8..5a9ae299d 100644
--- a/src/amdinfer/workers/worker.hpp
+++ b/src/amdinfer/workers/worker.hpp
@@ -31,6 +31,7 @@
 #include "amdinfer/batching/soft.hpp"
 #include "amdinfer/buffers/buffer.hpp"
 #include "amdinfer/build_options.hpp"
+#include "amdinfer/core/memory_pool/pool.hpp"
 #include "amdinfer/core/predict_api.hpp"
 #include "amdinfer/observation/logging.hpp"
 
@@ -43,12 +44,10 @@ namespace workers {
 enum class WorkerStatus {
   New,
   Init,
-  Allocate,
   Acquire,
   Run,
   Inactive,
   Release,
-  Deallocate,
   Destroy,
   Dead
 };
@@ -63,10 +62,6 @@ class Worker {
   Worker(const std::string& name, const std::string& platform)
     : metadata_(name, platform) {
     this->status_ = WorkerStatus::New;
-    this->input_buffers_ = nullptr;
-    this->output_buffers_ = nullptr;
-    this->max_buffer_num_ = UINT_MAX;
-    this->batch_size_ = 1;
   }
   virtual ~Worker() = default;  ///< Destroy the Worker object
 
@@ -78,16 +73,14 @@ class Worker {
    */
   virtual std::thread spawn(BatchPtrQueue* input_queue) = 0;
 
+  /// Allocate some buffers that are used to hold input and output data
+  [[nodiscard]] virtual std::vector<MemoryAllocators> getAllocators() const = 0;
+
   /// Perform low-cost initialization of the worker
   void init(ParameterMap* parameters) {
     this->status_ = WorkerStatus::Init;
     this->doInit(parameters);
   }
-  /// Allocate some buffers that are used to hold input and output data
-  size_t allocate(size_t num) {
-    this->status_ = WorkerStatus::Allocate;
-    return this->doAllocate(num);
-  }
   /// Acquire any hardware resources or perform high-cost initialization
   void acquire(ParameterMap* parameters) {
     this->status_ = WorkerStatus::Acquire;
@@ -110,13 +103,6 @@ class Worker {
     this->metadata_.setReady(false);
     this->doRelease();
   }
-  /// Free the input and output buffers
-  void deallocate() {
-    this->status_ = WorkerStatus::Deallocate;
-    this->doDeallocate();
-    this->input_buffers_ = nullptr;
-    this->output_buffers_ = nullptr;
-  }
   /// Perform any final operations before the worker thread is joined
   void destroy() {
     this->status_ = WorkerStatus::Destroy;
@@ -124,46 +110,22 @@ class Worker {
     this->status_ = WorkerStatus::Dead;
   }
 
-  /**
-   * @brief Return buffers to the worker's buffer pool after use. The buffers'
-   * reset() method is called prior to returning.
-   *
-   * @param input_buffers
-   * @param output_buffers
-   */
-  void returnBuffers(std::unique_ptr<std::vector<BufferPtrs>> input_buffers,
-                     std::unique_ptr<std::vector<BufferPtrs>> output_buffers) {
-    this->input_buffers_->enqueue_bulk(
-      std::make_move_iterator(input_buffers->begin()), input_buffers->size());
-    this->output_buffers_->enqueue_bulk(
-      std::make_move_iterator(output_buffers->begin()), output_buffers->size());
-  }
-
-  void setInputBuffers(BufferPtrsQueue* buffers) {
-    this->input_buffers_ = buffers;
-  }
-
-  void setOutputBuffers(BufferPtrsQueue* buffers) {
-    this->output_buffers_ = buffers;
-  }
-
-  [[nodiscard]] uint32_t getMaxBufferNum() const {
-    return this->max_buffer_num_;
-  }
+  void setPool(MemoryPool* pool) { pool_ = pool; }
 
   [[nodiscard]] size_t getBatchSize() const { return this->batch_size_; }
   [[nodiscard]] WorkerStatus getStatus() const { return this->status_; }
 
   virtual std::vector<std::unique_ptr<Batcher>> makeBatcher(
-    int num, ParameterMap* parameters) {
-    return this->makeBatcher<SoftBatcher>(num, parameters);
+    int num, ParameterMap* parameters, MemoryPool* pool) {
+    return this->makeBatcher<SoftBatcher>(num, parameters, pool);
   }
 
   template <typename T>
   std::vector<std::unique_ptr<Batcher>> makeBatcher(int num,
-                                                    ParameterMap* parameters) {
+                                                    ParameterMap* parameters,
+                                                    MemoryPool* pool) {
     std::vector<std::unique_ptr<Batcher>> batchers;
-    batchers.emplace_back(std::make_unique<T>(parameters));
+    batchers.emplace_back(std::make_unique<T>(pool, parameters));
     for (int i = 1; i < num; i++) {
       batchers.push_back(
         std::make_unique<T>(*dynamic_cast<T*>(batchers.back().get())));
@@ -178,17 +140,20 @@ class Worker {
   [[nodiscard]] const Logger& getLogger() const { return logger_; };
 #endif
 
-  BufferPtrsQueue* input_buffers_;
-  BufferPtrsQueue* output_buffers_;
-  uint32_t max_buffer_num_;
-  size_t batch_size_;
+  void returnInputBuffers(std::unique_ptr<Batch> batch) {
+    auto buffers = batch->getInputBuffers();
+    for (auto& buffer : buffers) {
+      pool_->put(std::move(buffer));
+    }
+  }
+
+  size_t batch_size_ = 1;
   ModelMetadata metadata_;
+  MemoryPool* pool_;
 
  private:
   /// Perform low-cost initialization of the worker
   virtual void doInit(ParameterMap* parameters) = 0;
-  /// Allocate some buffers that are used to hold input and output data
-  virtual size_t doAllocate(size_t num) = 0;
   /// Acquire any hardware resources or perform high-cost initialization
   virtual void doAcquire(ParameterMap* parameters) = 0;
   /**
@@ -199,8 +164,6 @@ class Worker {
   virtual void doRun(BatchPtrQueue* input_queue) = 0;
   /// Release any hardware resources
   virtual void doRelease() = 0;
-  /// Free the input and output buffers
-  virtual void doDeallocate() = 0;
   /// Perform any final operations before the worker's run thread is joined
   virtual void doDestroy() = 0;
 
diff --git a/src/amdinfer/workers/xmodel.cpp b/src/amdinfer/workers/xmodel.cpp
index 54b79cfce..c5c6efc95 100644
--- a/src/amdinfer/workers/xmodel.cpp
+++ b/src/amdinfer/workers/xmodel.cpp
@@ -42,24 +42,23 @@
 #include <xir/graph/subgraph.hpp>       // for Subgraph
 #include <xir/tensor/tensor.hpp>        // for Tensor
 
-#include "amdinfer/batching/batcher.hpp"            // for BatchPtr, Batch
-#include "amdinfer/buffers/buffer.hpp"              // for Buffer
-#include "amdinfer/buffers/vart_tensor_buffer.hpp"  // for VartTensorBuffer
-#include "amdinfer/build_options.hpp"               // for AMDINFER_ENABLE_ME...
-#include "amdinfer/core/data_types.hpp"             // for DataType
-#include "amdinfer/core/data_types_internal.hpp"    // for mapXirToType
-#include "amdinfer/core/exceptions.hpp"             // for invalid_argument
-#include "amdinfer/core/parameters.hpp"             // for ParameterMap
-#include "amdinfer/core/predict_api.hpp"            // for InferenceResponse
-#include "amdinfer/declarations.hpp"                // for BufferPtrs, Infere...
-#include "amdinfer/observation/observer.hpp"        // for Loggers, Metrics...
-#include "amdinfer/util/containers.hpp"             // for containerProduct
-#include "amdinfer/util/ctpl.hpp"                   // for ThreadPool
-#include "amdinfer/util/parse_env.hpp"              // for autoExpandEnvironm...
-#include "amdinfer/util/queue.hpp"                  // for BufferPtrsQueue
-#include "amdinfer/util/thread.hpp"                 // for setThreadName
-#include "amdinfer/util/timer.hpp"                  // for Timer
-#include "amdinfer/workers/worker.hpp"              // for Worker, kNumBuffer...
+#include "amdinfer/batching/batcher.hpp"          // for BatchPtr, Batch
+#include "amdinfer/buffers/buffer.hpp"            // for Buffer
+#include "amdinfer/build_options.hpp"             // for AMDINFER_ENABLE_ME...
+#include "amdinfer/core/data_types.hpp"           // for DataType
+#include "amdinfer/core/data_types_internal.hpp"  // for mapXirToType
+#include "amdinfer/core/exceptions.hpp"           // for invalid_argument
+#include "amdinfer/core/parameters.hpp"           // for ParameterMap
+#include "amdinfer/core/predict_api.hpp"          // for InferenceResponse
+#include "amdinfer/declarations.hpp"              // for BufferPtrs, Infere...
+#include "amdinfer/observation/observer.hpp"      // for Loggers, Metrics...
+#include "amdinfer/util/containers.hpp"           // for containerProduct
+#include "amdinfer/util/ctpl.hpp"                 // for ThreadPool
+#include "amdinfer/util/parse_env.hpp"            // for autoExpandEnvironm...
+#include "amdinfer/util/queue.hpp"                // for BufferPtrsQueue
+#include "amdinfer/util/thread.hpp"               // for setThreadName
+#include "amdinfer/util/timer.hpp"                // for Timer
+#include "amdinfer/workers/worker.hpp"            // for Worker, kNumBuffer...
 
 namespace amdinfer::workers {
 
@@ -73,14 +72,13 @@ class XModel : public Worker {
  public:
   XModel() : Worker("XModel", "XModel") {}
   std::thread spawn(BatchPtrQueue* input_queue) override;
+  [[nodiscard]] std::vector<MemoryAllocators> getAllocators() const override;
 
  private:
   void doInit(ParameterMap* parameters) override;
-  size_t doAllocate(size_t num) override;
   void doAcquire(ParameterMap* parameters) override;
   void doRun(BatchPtrQueue* input_queue) override;
   void doRelease() override;
-  void doDeallocate() override;
   void doDestroy() override;
 
   vart::RunnerExt* getRunner();
@@ -91,19 +89,22 @@ class XModel : public Worker {
   std::unique_ptr<vart::Runner> runner_;
   std::vector<DataType> output_type_;
   std::vector<uint32_t> output_size_;
-  util::ThreadPool pool_;
+  util::ThreadPool thread_pool_;
 };
 
 std::thread XModel::spawn(BatchPtrQueue* input_queue) {
   return std::thread(&XModel::run, this, input_queue);
 }
 
+std::vector<MemoryAllocators> XModel::getAllocators() const {
+  return {MemoryAllocators::Cpu};
+}
+
 vart::RunnerExt* XModel::getRunner() {
   return dynamic_cast<vart::RunnerExt*>(this->runner_.get());
 }
 
 void XModel::doInit(ParameterMap* parameters) {
-  const auto max_buffer_num = 50;
   const auto* aks_xmodel_root = std::getenv("AKS_XMODEL_ROOT");
   if (aks_xmodel_root == nullptr) {
     throw environment_not_set_error("AKS_XMODEL_ROOT not set");
@@ -112,11 +113,6 @@ void XModel::doInit(ParameterMap* parameters) {
     std::string{aks_xmodel_root} +
     "/artifacts/u200_u250/resnet_v1_50_tf/resnet_v1_50_tf.xmodel";
 
-  max_buffer_num_ = max_buffer_num;
-  if (parameters->has("max_buffer_num")) {
-    max_buffer_num_ = parameters->get<int32_t>("max_buffer_num");
-  }
-
   auto path = default_path;
   if (parameters->has("model")) {
     path = parameters->get<std::string>("model");
@@ -149,41 +145,6 @@ void XModel::doInit(ParameterMap* parameters) {
   }
 }
 
-size_t XModel::doAllocate(size_t num) {
-  constexpr auto kBufferNum = 10U;
-  size_t buffer_num =
-    static_cast<int>(num) == kNumBufferAuto ? kBufferNum : num;
-
-  for (size_t i = 0; i < buffer_num; i++) {
-    BufferPtrs vec;
-    auto input_tensors = subgraph_->get_sorted_input_tensors();
-    if (input_tensors.size() > 1) {
-      // rt-engine/vart does not support more than one input tensor
-      throw(
-        invalid_argument("Unsupported XModel with more than one input tensor"));
-    }
-    for (const auto* tensor : input_tensors) {
-      auto input_shape = tensor->get_shape();
-      auto input_type = tensor->get_data_type();
-      vec.emplace_back(std::make_unique<VartTensorBuffer>(
-        tensor->get_name(), input_shape, input_type));
-    }
-    this->input_buffers_->enqueue(std::move(vec));
-  }
-  for (size_t i = 0; i < buffer_num; i++) {
-    BufferPtrs vec;
-    auto output_tensors = subgraph_->get_sorted_output_tensors();
-    for (const auto* tensor : output_tensors) {
-      auto input_shape = tensor->get_shape();
-      auto input_type = tensor->get_data_type();
-      vec.emplace_back(std::make_unique<VartTensorBuffer>(
-        tensor->get_name(), input_shape, input_type));
-    }
-    this->output_buffers_->enqueue(std::move(vec));
-  }
-  return buffer_num;
-}
-
 void XModel::doAcquire(ParameterMap* parameters) {
   constexpr auto kThreads = 3;
 
@@ -191,7 +152,7 @@ void XModel::doAcquire(ParameterMap* parameters) {
   if (parameters->has("threads")) {
     threads = parameters->get<int32_t>("threads");
   }
-  this->pool_.resize(threads);
+  this->thread_pool_.resize(threads);
 
   runner_ = vart::Runner::create_runner(this->subgraph_, "run");
   auto input_tensors = runner_->get_input_tensors();
@@ -215,14 +176,15 @@ void XModel::doAcquire(ParameterMap* parameters) {
 }
 
 void XModel::doRun(BatchPtrQueue* input_queue) {
-  std::atomic_int32_t pool_size = 0;
-  const int max_pool_size = this->pool_.getSize() * 4;  // 4 is arbitrary
+  std::atomic_int32_t thread_pool_size = 0;
+  const int max_thread_pool_size =
+    this->thread_pool_.getSize() * 4;  // 4 is arbitrary
   util::setThreadName("XModel");
 #ifdef AMDINFER_ENABLE_LOGGING
   const auto& logger = this->getLogger();
 #endif
 
-  const auto pool_delay = std::chrono::milliseconds(10);
+  const auto thread_pool_delay = std::chrono::milliseconds(10);
 
   while (true) {
     BatchPtr batch;
@@ -236,136 +198,146 @@ void XModel::doRun(BatchPtrQueue* input_queue) {
     Metrics::getInstance().incrementCounter(
       MetricCounterIDs::PipelineIngressWorker);
 #endif
-    pool_size++;
-    if (pool_size > max_pool_size) {
-      std::this_thread::sleep_for(pool_delay);
+    thread_pool_size++;
+    if (thread_pool_size > max_thread_pool_size) {
+      std::this_thread::sleep_for(thread_pool_delay);
     }
-    this->pool_.push([this, batch = std::move(batch), &pool_size](int id) {
-      (void)id;  // suppress unused variable warning
+    this->thread_pool_.push(
+      [this, batch = std::move(batch), &thread_pool_size](int id) {
+        (void)id;  // suppress unused variable warning
 #ifdef AMDINFER_ENABLE_TRACING
-      for (unsigned int j = 0; j < batch->size(); j++) {
-        auto& trace = batch->getTrace(j);
-        trace->startSpan("xmodel");
-      }
+        for (unsigned int j = 0; j < batch->size(); j++) {
+          auto& trace = batch->getTrace(j);
+          trace->startSpan("xmodel");
+        }
 #endif
 
-      std::queue<std::pair<uint32_t, int>> futures;
-      std::vector<vart::TensorBuffer*> inputs_ptr;
-      std::vector<vart::TensorBuffer*> outputs_ptr;
-
-      const auto& input_buffers = batch->getInputBuffers();
-      inputs_ptr.reserve(input_buffers.size());
-      const auto& output_buffers = batch->getOutputBuffers();
-      outputs_ptr.reserve(output_buffers.size());
-
-      for (const auto& buffer : input_buffers) {
-        logTraceBuffer(getLogger(), buffer->data(0));
-      }
-
-      for (const auto& buffer : input_buffers) {
-        auto* vart = dynamic_cast<VartTensorBuffer*>(buffer.get());
-        inputs_ptr.emplace_back(vart->getTensorBuffer());
-      }
-      for (const auto& buffer : output_buffers) {
-        auto* vart = dynamic_cast<VartTensorBuffer*>(buffer.get());
-        outputs_ptr.emplace_back(vart->getTensorBuffer());
-      }
-
-      for (auto* input : inputs_ptr) {
-        const auto* tensor = input->get_tensor();
-        input->sync_for_write(
-          0, tensor->get_element_num() / (tensor->get_shape())[0]);
-      }
-
-      futures.push(getRunner()->execute_async(inputs_ptr, outputs_ptr));
-
-      std::vector<InferenceResponse> responses;
-      responses.reserve(batch->size());
-
-      for (const auto& req : *batch) {
-        auto& resp = responses.emplace_back();
-        resp.setID(req->getID());
-        resp.setModel("xmodel");
-      }
-
-      while (!futures.empty()) {
-        auto job_id = futures.front();
-        futures.pop();
-        getRunner()->wait(job_id.first, -1);
-      }
-
-      for (auto* output : outputs_ptr) {
-        const auto* tensor = output->get_tensor();
-        output->sync_for_read(
-          0, tensor->get_element_num() / (tensor->get_shape())[0]);
-      }
-
-      const auto num_batches = batch->size();
-      for (unsigned int k = 0; k < num_batches; k++) {
-        const auto& req = batch->getRequest(k);
-        auto inputs = req->getInputs();
-        auto outputs = req->getOutputs();
-        auto& resp = responses[k];
-
-        const auto num_outputs = outputs.size();
-        for (unsigned int i = 0; i < num_outputs; i++) {
-          auto* output_index = output_buffers[i]->data(0);
-          InferenceResponseOutput output;
-          auto output_tensors = getRunner()->get_output_tensors();
-          auto output_shape = output_tensors[i]->get_shape();
-          std::vector<uint64_t> new_shape;
-          new_shape.reserve(output_shape.size() - 1);
-          for (size_t j = 0; j < output_shape.size() - 1; j++) {
-            new_shape.push_back(output_shape[j + 1]);
-          }
-          output.setShape(new_shape);
+        std::queue<std::pair<uint32_t, int>> futures;
+
+        auto input_buffers = batch->getInputBuffers();
+        // const auto& output_buffers = batch->getOutputBuffers();
+
+        for (const auto& buffer : input_buffers) {
+          logTraceBuffer(getLogger(), buffer->data(0));
+        }
 
-          output.setDatatype(this->output_type_[i]);
+        auto inputs_ptr = getRunner()->get_inputs();
+        if (inputs_ptr.size() != input_buffers.size()) {
+          throw invalid_argument("Input size does not match model");
+        }
+        for (auto j = 0U; j < input_buffers.size(); ++j) {
+          auto* tensor = inputs_ptr.at(j);
+          auto& buffer = input_buffers.at(j);
 
-          std::vector<std::byte> buffer;
-          buffer.resize(this->output_size_[i] * sizeof(uint8_t));
-          memcpy(buffer.data(),
-                 reinterpret_cast<int8_t*>(output_index) +
-                   (i * this->output_size_[i]),
-                 this->output_size_[i] * sizeof(uint8_t));
-          output.setData(std::move(buffer));
+          const auto [addr, size] = tensor->data();
+          auto* tensor_addr = reinterpret_cast<void*>(addr);
 
-          std::string output_name = outputs[i].getName();
+          std::memcpy(tensor_addr, buffer->data(0), size);
+        }
+        auto outputs_ptr = getRunner()->get_outputs();
 
-          if (output_name.empty()) {
-            output.setName(inputs[0].getName());
-          } else {
-            output.setName(output_name);
-          }
+        for (auto* input : inputs_ptr) {
+          const auto* tensor = input->get_tensor();
+          input->sync_for_write(
+            0, tensor->get_element_num() / (tensor->get_shape())[0]);
+        }
+
+        futures.push(getRunner()->execute_async(inputs_ptr, outputs_ptr));
 
-          resp.addOutput(output);
+        std::vector<InferenceResponse> responses;
+        responses.reserve(batch->size());
+
+        for (const auto& req : *batch) {
+          auto& resp = responses.emplace_back();
+          resp.setID(req->getID());
+          resp.setModel("xmodel");
         }
 
+        while (!futures.empty()) {
+          auto job_id = futures.front();
+          futures.pop();
+          getRunner()->wait(job_id.first, -1);
+        }
+
+        for (auto* output : outputs_ptr) {
+          const auto* tensor = output->get_tensor();
+          output->sync_for_read(
+            0, tensor->get_element_num() / (tensor->get_shape())[0]);
+        }
+
+        const auto num_batches = batch->size();
+        for (unsigned int k = 0; k < num_batches; k++) {
+          const auto& req = batch->getRequest(k);
+          auto inputs = req->getInputs();
+          auto outputs = req->getOutputs();
+          auto& resp = responses[k];
+
+          const auto num_outputs = outputs_ptr.size();
+          for (unsigned int i = 0; i < num_outputs; i++) {
+            // auto* output_index = output_buffers[i]->data(0);
+            auto* output_index =
+              reinterpret_cast<void*>(outputs_ptr.at(i)->data().first);
+            InferenceResponseOutput output;
+            auto output_tensors = getRunner()->get_output_tensors();
+            auto output_shape = output_tensors[i]->get_shape();
+            std::vector<uint64_t> new_shape;
+            new_shape.reserve(output_shape.size() - 1);
+            for (size_t j = 0; j < output_shape.size() - 1; j++) {
+              new_shape.push_back(output_shape[j + 1]);
+            }
+            output.setShape(new_shape);
+
+            output.setDatatype(this->output_type_[i]);
+
+            std::vector<std::byte> buffer;
+            buffer.resize(this->output_size_[i] * sizeof(uint8_t));
+            memcpy(buffer.data(),
+                   reinterpret_cast<int8_t*>(output_index) +
+                     (i * this->output_size_[i]),
+                   this->output_size_[i] * sizeof(uint8_t));
+            output.setData(std::move(buffer));
+
+            std::string output_name;
+            if (i < outputs.size()) {
+              output_name = outputs[i].getName();
+            }
+
+            if (output_name.empty()) {
+              output.setName(inputs[0].getName());
+            } else {
+              output.setName(output_name);
+            }
+
+            resp.addOutput(output);
+          }
+
 #ifdef AMDINFER_ENABLE_TRACING
-        auto context = batch->getTrace(k)->propagate();
-        resp.setContext(std::move(context));
+          auto context = batch->getTrace(k)->propagate();
+          resp.setContext(std::move(context));
 #endif
 
-        req->runCallbackOnce(resp);
+          req->runCallbackOnce(resp);
 #ifdef AMDINFER_ENABLE_METRICS
-        Metrics::getInstance().incrementCounter(
-          MetricCounterIDs::PipelineEgressWorker);
-        util::Timer timer{batch->getTime(k)};
-        timer.stop();
-        auto duration = timer.count<std::micro>();
-        Metrics::getInstance().observeSummary(MetricSummaryIDs::RequestLatency,
-                                              duration);
+          Metrics::getInstance().incrementCounter(
+            MetricCounterIDs::PipelineEgressWorker);
+          util::Timer timer{batch->getTime(k)};
+          timer.stop();
+          auto duration = timer.count<std::micro>();
+          Metrics::getInstance().observeSummary(
+            MetricSummaryIDs::RequestLatency, duration);
 #endif
-      }
-      pool_size--;
-    });
+        }
+        for (auto& buffer : input_buffers) {
+          pool_->put(std::move(buffer));
+        }
+        thread_pool_size--;
+      });
   }
   AMDINFER_LOG_INFO(logger, "XModel ending");
 }
 
 void XModel::doRelease() {}
-void XModel::doDeallocate() { this->pool_.stop(true); }
-void XModel::doDestroy() {}
+void XModel::doDestroy() { this->thread_pool_.stop(true); }
 
 }  // namespace amdinfer::workers
 


Test patch:
diff --git a/tests/api/test_infer_async.cpp b/tests/api/test_infer_async.cpp
index 7060f06f3..bfa9a7d8c 100644
--- a/tests/api/test_infer_async.cpp
+++ b/tests/api/test_infer_async.cpp
@@ -23,6 +23,7 @@
 
 namespace amdinfer {
 
+#ifdef AMDINFER_ENABLE_HTTP
 // NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
 TEST_F(HttpFixture, Ordered) {
   NativeClient client(&server_);
@@ -51,5 +52,6 @@ TEST_F(HttpFixture, Ordered) {
     EXPECT_FALSE(resp.isError());
   }
 }
+#endif
 
 }  // namespace amdinfer
diff --git a/tests/performance/batching/CMakeLists.txt b/tests/performance/batching/CMakeLists.txt
index 14659fee9..0371c8cca 100644
--- a/tests/performance/batching/CMakeLists.txt
+++ b/tests/performance/batching/CMakeLists.txt
@@ -20,8 +20,8 @@ list(
     tests_libs
     "$<TARGET_OBJECTS:batcher>~fake_observation~interface\
       ~$<TARGET_OBJECTS:fake_worker_info_buffers_finite>~predict_api~parameters\
-      ~data_types\~$<TARGET_OBJECTS:softBatcher>~vector_buffer~buffer\
-      ~$<TARGET_OBJECTS:native_internal>~timer"
+      ~data_types\~$<TARGET_OBJECTS:softBatcher>~cpuBuffer~buffer~memory_pool\
+      ~batch~$<TARGET_OBJECTS:native_internal>~timer"
 )
 
 amdinfer_add_benchmarks("${tests}" "${tests_libs}")
diff --git a/tests/performance/batching/benchmark_soft.cpp b/tests/performance/batching/benchmark_soft.cpp
index c8de44559..74ba9c36a 100644
--- a/tests/performance/batching/benchmark_soft.cpp
+++ b/tests/performance/batching/benchmark_soft.cpp
@@ -31,9 +31,9 @@
 #include <vector>    // for vector
 
 #include "amdinfer/batching/soft.hpp"            // for BatchPtr, SoftBatcher
-#include "amdinfer/buffers/vector_buffer.hpp"    // for VectorBuffer
 #include "amdinfer/clients/native_internal.hpp"  // for CppNativeApi
 #include "amdinfer/core/data_types.hpp"          // for DataType, DataType::U...
+#include "amdinfer/core/memory_pool/pool.hpp"    // for MemoryPool
 #include "amdinfer/core/parameters.hpp"          // for ParameterMap
 #include "amdinfer/core/predict_api.hpp"         // for InferenceRequest, Req...
 #include "amdinfer/core/worker_info.hpp"         // for WorkerInfo
@@ -64,7 +64,6 @@ class PerfSoftBatcherFixture : public ::benchmark::Fixture {
 
   void SetUp(const ::benchmark::State& state) override {
     auto batch_size = static_cast<int>(state.range(0));
-    auto buffer_num = static_cast<size_t>(state.range(1));
 
     const auto data_shape = {1UL, 2UL, 50UL};
     const auto data_size = 100;
@@ -79,25 +78,25 @@ class PerfSoftBatcherFixture : public ::benchmark::Fixture {
     options.file_enable = false;
     initLogger(options);
 
-    this->batcher_.emplace(&parameters);
+    this->batcher_.emplace(&pool_, &parameters);
     this->batcher_->setName("test");
     this->batcher_->setBatchSize(batch_size);
 
-    this->worker_.emplace("", &parameters);
-    for (size_t i = 0; i < buffer_num; i++) {
-      BufferPtrs vec;
-      vec.emplace_back(std::make_unique<VectorBuffer>(batch_size * data_size,
-                                                      DataType::Uint8));
-      this->worker_->putInputBuffer(std::move(vec));
-    }
-    for (size_t i = 0; i < buffer_num; i++) {
-      BufferPtrs vec;
-      vec.emplace_back(std::make_unique<VectorBuffer>(batch_size * data_size,
-                                                      DataType::Uint8));
-      this->worker_->putOutputBuffer(std::move(vec));
-    }
-
-    this->batcher_->start(&this->worker_.value());
+    this->worker_.emplace("", &parameters, &pool_);
+    // for (size_t i = 0; i < buffer_num; i++) {
+    //   BufferPtrs vec;
+    //   vec.emplace_back(std::make_unique<VectorBuffer>(batch_size * data_size,
+    //                                                   DataType::Uint8));
+    //   this->worker_->putInputBuffer(std::move(vec));
+    // }
+    // for (size_t i = 0; i < buffer_num; i++) {
+    //   BufferPtrs vec;
+    //   vec.emplace_back(std::make_unique<VectorBuffer>(batch_size * data_size,
+    //                                                   DataType::Uint8));
+    //   this->worker_->putOutputBuffer(std::move(vec));
+    // }
+
+    this->batcher_->start({MemoryAllocators::Cpu});
 
     data_.resize(data_size);
     for (auto i = 0; i < data_size; i++) {
@@ -119,6 +118,7 @@ class PerfSoftBatcherFixture : public ::benchmark::Fixture {
   InferenceRequest request_;
   std::optional<WorkerInfo> worker_;
   std::optional<SoftBatcher> batcher_;
+  MemoryPool pool_;
 };
 
 BENCHMARK_DEFINE_F(PerfSoftBatcherFixture, BasicBatching)
@@ -144,14 +144,12 @@ BENCHMARK_DEFINE_F(PerfSoftBatcherFixture, BasicBatching)
 
 const std::initializer_list<int64_t> kBatchSizes{1, 2, 4};
 
-const std::initializer_list<int64_t> kBufferNums{1, 10, 100};
-
 // delay after dequeue per request in microseconds
 const std::initializer_list<int64_t> kWorkDelay{0, 1, 10};
 
 // NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
 BENCHMARK_REGISTER_F(PerfSoftBatcherFixture, BasicBatching)
-  ->ArgsProduct({kBatchSizes, kBufferNums, kWorkDelay})
+  ->ArgsProduct({kBatchSizes, kWorkDelay})
   ->Unit(benchmark::kMillisecond);
 
 // NOLINTNEXTLINE
diff --git a/tests/src/amdinfer/CMakeLists.txt b/tests/src/amdinfer/CMakeLists.txt
index 6fed56d27..992ebb034 100644
--- a/tests/src/amdinfer/CMakeLists.txt
+++ b/tests/src/amdinfer/CMakeLists.txt
@@ -14,7 +14,8 @@
 # limitations under the License.
 
 add_subdirectory(batching)
-# add_subdirectory(buffers) add_subdirectory(clients)
+# add_subdirectory(buffers)
+# add_subdirectory(clients)
 add_subdirectory(core)
 add_subdirectory(observation)
 # add_subdirectory(servers)
diff --git a/tests/src/amdinfer/batching/batcher.cpp b/tests/src/amdinfer/batching/batcher.cpp
index 216f808f5..ca241dfb9 100644
--- a/tests/src/amdinfer/batching/batcher.cpp
+++ b/tests/src/amdinfer/batching/batcher.cpp
@@ -41,110 +41,111 @@
 
 namespace amdinfer {
 
-/**
- * @brief For testing purposes, this fake C++ interface returns a
- * fakeInferenceRequest object similar to how the real one does.
- *
- */
-class FakeInterface : public Interface {
- public:
-  explicit FakeInterface(InferenceRequestInput request);
-
-  std::shared_ptr<InferenceRequest> getRequest(
-    const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
-    const BufferRawPtrs &output_buffers,
-    std::vector<size_t> &output_offsets) override;
-
-  size_t getInputSize() override;
-  void errorHandler(const std::exception &e) override;
-  std::promise<amdinfer::InferenceResponse> *getPromise();
-
- private:
-  InferenceRequestInput request_;
-  InferenceResponsePromisePtr promise_;
-};
-
-FakeInterface::FakeInterface(InferenceRequestInput request)
-  : request_(std::move(request)) {
-  this->promise_ =
-    std::make_unique<std::promise<amdinfer::InferenceResponse>>();
-}
-
-size_t FakeInterface::getInputSize() { return 1; }
-
-std::promise<amdinfer::InferenceResponse> *FakeInterface::getPromise() {
-  return this->promise_.get();
-}
-
-void fakeCppCallback(const InferenceResponsePromisePtr &promise,
-                     const InferenceResponse &response) {
-  promise->set_value(response);
-}
-
-std::shared_ptr<InferenceRequest> FakeInterface::getRequest(
-  const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
-  const BufferRawPtrs &output_buffers, std::vector<size_t> &output_offsets) {
-  auto request = std::make_shared<FakeInferenceRequest>(
-    this->request_, input_buffers, input_offsets, output_buffers,
-    output_offsets);
-  Callback callback = [this](const InferenceResponse &response) {
-    fakeCppCallback(this->promise_, response);
-  };
-  request->setCallback(std::move(callback));
-  return request;
-}
-
-void FakeInterface::errorHandler(const std::exception &e) {
-  AMDINFER_LOG_ERROR(this->getLogger(), e.what());
-  (void)e;  // suppress unused variable warning
-}
-
-void Batcher::run(WorkerInfo *worker) {
-  auto thread_name = "batch" + this->getName();
-  util::setThreadName(thread_name);
-  InterfacePtr req;
-  bool run = true;
-
-  while (run) {
-    auto batch = std::make_unique<Batch>(worker);
-    auto input_buffers = batch->getRawInputBuffers();
-    auto output_buffers = batch->getRawOutputBuffers();
-    std::vector<size_t> input_offset(input_buffers.size(), 0);
-    std::vector<size_t> output_offset(output_buffers.size(), 0);
-
-    // wait for the first request
-    this->input_queue_->wait_dequeue(req);
-    AMDINFER_LOG_DEBUG(
-      logger_, "Got initial request of a new batch for " + this->model_);
-
-    if (req == nullptr) {
-      break;
-    }
-
-#ifdef AMDINFER_ENABLE_TRACING
-    auto trace = req->getTrace();
-    trace->startSpan("fake_batcher");
-#endif
-    req->getInputSize();  // initialize the req object
-    auto new_req = req->getRequest(input_buffers, input_offset, output_buffers,
-                                   output_offset);
-
-    batch->addRequest(new_req);
-
-#ifdef AMDINFER_ENABLE_TRACING
-    trace->endSpan();
-    batch->addTrace(std::move(trace));
-#endif
-#ifdef AMDINFER_ENABLE_METRICS
-    batch->addTime(req->getTime());
-#endif
-    // batch_size += 1;
-
-    if (!batch->empty()) {
-      AMDINFER_LOG_DEBUG(logger_, "Enqueuing batch for " + this->model_);
-      this->output_queue_->enqueue(std::move(batch));
-    }
-  }
-}
+// /**
+//  * @brief For testing purposes, this fake C++ interface returns a
+//  * fakeInferenceRequest object similar to how the real one does.
+//  *
+//  */
+// class FakeInterface : public Interface {
+//  public:
+//   explicit FakeInterface(InferenceRequestInput request);
+
+//   std::shared_ptr<InferenceRequest> getRequest(
+//     const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+//     const BufferRawPtrs &output_buffers,
+//     std::vector<size_t> &output_offsets) override;
+
+//   size_t getInputSize() override;
+//   void errorHandler(const std::exception &e) override;
+//   std::promise<amdinfer::InferenceResponse> *getPromise();
+
+//  private:
+//   InferenceRequestInput request_;
+//   InferenceResponsePromisePtr promise_;
+// };
+
+// FakeInterface::FakeInterface(InferenceRequestInput request)
+//   : request_(std::move(request)) {
+//   this->promise_ =
+//     std::make_unique<std::promise<amdinfer::InferenceResponse>>();
+// }
+
+// size_t FakeInterface::getInputSize() { return 1; }
+
+// std::promise<amdinfer::InferenceResponse> *FakeInterface::getPromise() {
+//   return this->promise_.get();
+// }
+
+// void fakeCppCallback(const InferenceResponsePromisePtr &promise,
+//                      const InferenceResponse &response) {
+//   promise->set_value(response);
+// }
+
+// std::shared_ptr<InferenceRequest> FakeInterface::getRequest(
+//   const BufferRawPtrs &input_buffers, std::vector<size_t> &input_offsets,
+//   const BufferRawPtrs &output_buffers, std::vector<size_t> &output_offsets) {
+//   auto request = std::make_shared<FakeInferenceRequest>(
+//     this->request_, input_buffers, input_offsets, output_buffers,
+//     output_offsets);
+//   Callback callback = [this](const InferenceResponse &response) {
+//     fakeCppCallback(this->promise_, response);
+//   };
+//   request->setCallback(std::move(callback));
+//   return request;
+// }
+
+// void FakeInterface::errorHandler(const std::exception &e) {
+//   AMDINFER_LOG_ERROR(this->getLogger(), e.what());
+//   (void)e;  // suppress unused variable warning
+// }
+
+// void Batcher::run(WorkerInfo *worker) {
+//   auto thread_name = "batch" + this->getName();
+//   util::setThreadName(thread_name);
+//   InterfacePtr req;
+//   bool run = true;
+
+//   while (run) {
+//     auto batch = std::make_unique<Batch>(worker);
+//     auto input_buffers = batch->getRawInputBuffers();
+//     auto output_buffers = batch->getRawOutputBuffers();
+//     std::vector<size_t> input_offset(input_buffers.size(), 0);
+//     std::vector<size_t> output_offset(output_buffers.size(), 0);
+
+//     // wait for the first request
+//     this->input_queue_->wait_dequeue(req);
+//     AMDINFER_LOG_DEBUG(
+//       logger_, "Got initial request of a new batch for " + this->model_);
+
+//     if (req == nullptr) {
+//       break;
+//     }
+
+// #ifdef AMDINFER_ENABLE_TRACING
+//     auto trace = req->getTrace();
+//     trace->startSpan("fake_batcher");
+// #endif
+//     req->getInputSize();  // initialize the req object
+//     auto new_req = req->getRequest(input_buffers, input_offset,
+//     output_buffers,
+//                                    output_offset);
+
+//     batch->addRequest(new_req);
+
+// #ifdef AMDINFER_ENABLE_TRACING
+//     trace->endSpan();
+//     batch->addTrace(std::move(trace));
+// #endif
+// #ifdef AMDINFER_ENABLE_METRICS
+//     batch->addTime(req->getTime());
+// #endif
+//     // batch_size += 1;
+
+//     if (!batch->empty()) {
+//       AMDINFER_LOG_DEBUG(logger_, "Enqueuing batch for " + this->model_);
+//       this->output_queue_->enqueue(std::move(batch));
+//     }
+//   }
+// }
 
 }  // namespace amdinfer
diff --git a/tests/src/amdinfer/core/CMakeLists.txt b/tests/src/amdinfer/core/CMakeLists.txt
index ae37fbb20..61f9120cd 100644
--- a/tests/src/amdinfer/core/CMakeLists.txt
+++ b/tests/src/amdinfer/core/CMakeLists.txt
@@ -15,7 +15,6 @@
 
 list(
   APPEND targets
-         # predict_api
          predict_api
          worker_info_buffers_infinite
          worker_info_buffers_finite
diff --git a/tests/src/amdinfer/core/worker_info_buffers_finite.cpp b/tests/src/amdinfer/core/worker_info_buffers_finite.cpp
index 722a55b2a..712688185 100644
--- a/tests/src/amdinfer/core/worker_info_buffers_finite.cpp
+++ b/tests/src/amdinfer/core/worker_info_buffers_finite.cpp
@@ -35,24 +35,19 @@
 
 namespace amdinfer {
 
-WorkerInfo::WorkerInfo(const std::string& name, ParameterMap* parameters) {
-  // arbitrarily set to 10
-  const int max_buffer_num = 10;
-
-  this->input_buffer_ptr_ = std::make_unique<BufferPtrsQueue>();
-  this->output_buffer_ptr_ = std::make_unique<BufferPtrsQueue>();
-  this->buffer_num_ = max_buffer_num;
-  this->max_buffer_num_ = max_buffer_num;
+WorkerInfo::WorkerInfo(const std::string& name, ParameterMap* parameters,
+                       MemoryPool* pool) {
   this->batch_size_ = 1;
 
-  this->addAndStartWorker(name, parameters);
+  this->addAndStartWorker(name, parameters, pool);
 }
 
 WorkerInfo::~WorkerInfo() = default;
 
 void WorkerInfo::addAndStartWorker(const std::string& name,
-                                   ParameterMap* parameters) {
+                                   ParameterMap* parameters, MemoryPool* pool) {
   (void)name;
+  (void)pool;
   if (parameters == nullptr) {
     return;
   }
@@ -76,31 +71,4 @@ size_t WorkerInfo::getGroupSize() const { return 1; }
 
 void WorkerInfo::shutdown() {}
 
-BufferPtrs WorkerInfo::getInputBuffer() const {
-  BufferPtrs buffer;
-  input_buffer_ptr_->wait_dequeue(buffer);
-  return buffer;
-}
-
-BufferPtrs WorkerInfo::getOutputBuffer() const {
-  BufferPtrs buffer;
-  output_buffer_ptr_->wait_dequeue(buffer);
-  return buffer;
-}
-
-void WorkerInfo::putInputBuffer(BufferPtrs&& buffer) const {
-  this->input_buffer_ptr_->enqueue(std::move(buffer));
-}
-
-void WorkerInfo::putOutputBuffer(BufferPtrs&& buffer) const {
-  this->output_buffer_ptr_->enqueue(std::move(buffer));
-}
-
-// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-bool WorkerInfo::inputSizeValid([[maybe_unused]] size_t size) const {
-  return true;
-}
-
-void WorkerInfo::allocate([[maybe_unused]] size_t request_size) {}
-
 }  // namespace amdinfer
diff --git a/tests/src/amdinfer/core/worker_info_buffers_infinite.cpp b/tests/src/amdinfer/core/worker_info_buffers_infinite.cpp
index dfdde9085..c64d03ef4 100644
--- a/tests/src/amdinfer/core/worker_info_buffers_infinite.cpp
+++ b/tests/src/amdinfer/core/worker_info_buffers_infinite.cpp
@@ -34,23 +34,19 @@
 
 namespace amdinfer {
 
-constexpr auto kDefaultBufferNum = 10;
-
-WorkerInfo::WorkerInfo(const std::string& name, ParameterMap* parameters) {
-  this->input_buffer_ptr_ = std::make_unique<BufferPtrsQueue>();
-  this->output_buffer_ptr_ = std::make_unique<BufferPtrsQueue>();
-  this->buffer_num_ = kDefaultBufferNum;
-  this->max_buffer_num_ = kDefaultBufferNum;
+WorkerInfo::WorkerInfo(const std::string& name, ParameterMap* parameters,
+                       MemoryPool* pool) {
   this->batch_size_ = 1;
 
-  this->addAndStartWorker(name, parameters);
+  this->addAndStartWorker(name, parameters, pool);
 }
 
 WorkerInfo::~WorkerInfo() = default;
 
 void WorkerInfo::addAndStartWorker(const std::string& name,
-                                   ParameterMap* parameters) {
+                                   ParameterMap* parameters, MemoryPool* pool) {
   (void)name;
+  (void)pool;
   if (parameters == nullptr) {
     return;
   }
@@ -74,29 +70,4 @@ size_t WorkerInfo::getGroupSize() const { return 1; }
 
 void WorkerInfo::shutdown() {}
 
-// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-BufferPtrs WorkerInfo::getInputBuffer() const {
-  BufferPtrs buffer;
-  return buffer;
-}
-
-// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-BufferPtrs WorkerInfo::getOutputBuffer() const {
-  BufferPtrs buffer;
-  return buffer;
-}
-
-// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-void WorkerInfo::putInputBuffer([[maybe_unused]] BufferPtrs&& buffer) const {}
-
-// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-void WorkerInfo::putOutputBuffer([[maybe_unused]] BufferPtrs&& buffer) const {}
-
-// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-bool WorkerInfo::inputSizeValid([[maybe_unused]] size_t size) const {
-  return true;
-}
-
-void WorkerInfo::allocate([[maybe_unused]] size_t request_size) {}
-
 }  // namespace amdinfer
diff --git a/tests/src/amdinfer/testing/gtest.hpp b/tests/src/amdinfer/testing/gtest.hpp
new file mode 100644
index 000000000..58f0b70f6
--- /dev/null
+++ b/tests/src/amdinfer/testing/gtest.hpp
@@ -0,0 +1,39 @@
+// Copyright 2022 Xilinx, Inc.
+// Copyright 2022 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#ifndef GUARD_SRC_AMDINFER_TESTING_GTEST
+#define GUARD_SRC_AMDINFER_TESTING_GTEST
+
+#include "gtest/gtest.h"  // IWYU pragma: export
+
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define EXPECT_THROW_CHECK(statement, check, exception) \
+  EXPECT_THROW(                                         \
+    {                                                   \
+      try {                                             \
+        statement                                       \
+      } catch (const exception& e) {                    \
+        check;                                          \
+        throw;                                          \
+      }                                                 \
+    },                                                  \
+    exception)
+
+#endif  // GUARD_SRC_AMDINFER_TESTING_GTEST
diff --git a/tests/src/amdinfer/testing/gtest_fixtures.hpp b/tests/src/amdinfer/testing/gtest_fixtures.hpp
index b53e770bf..4d0baa5dd 100644
--- a/tests/src/amdinfer/testing/gtest_fixtures.hpp
+++ b/tests/src/amdinfer/testing/gtest_fixtures.hpp
@@ -100,17 +100,4 @@ template <typename T>
 class HttpFixtureWithParams : public HttpFixture,
                               public testing::WithParamInterface<T> {};
 
-// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
-#define EXPECT_THROW_CHECK(statement, check, exception) \
-  EXPECT_THROW(                                         \
-    {                                                   \
-      try {                                             \
-        statement                                       \
-      } catch (const exception& e) {                    \
-        check;                                          \
-        throw;                                          \
-      }                                                 \
-    },                                                  \
-    exception)
-
 #endif  // GUARD_SRC_AMDINFER_TESTING_GTEST_FIXTURES
diff --git a/tests/unit/batching/CMakeLists.txt b/tests/unit/batching/CMakeLists.txt
index 1a0cf99f5..4df173f35 100644
--- a/tests/unit/batching/CMakeLists.txt
+++ b/tests/unit/batching/CMakeLists.txt
@@ -20,11 +20,11 @@ list(
     tests_libs
     "$<TARGET_OBJECTS:batcher>~fake_observation~interface~\
       $<TARGET_OBJECTS:fake_worker_info_buffers_infinite>~predict_api~\
-      parameters~data_types~$<TARGET_OBJECTS:softBatcher>~timer"
+      parameters~data_types~$<TARGET_OBJECTS:softBatcher>~timer~memory_pool~batch~cpuBuffer~buffer"
     "$<TARGET_OBJECTS:batcher>~fake_observation~interface~\
       $<TARGET_OBJECTS:fake_worker_info_buffers_finite>~predict_api~data_types~\
-      parameters~$<TARGET_OBJECTS:softBatcher>~vector_buffer~buffer~\
-      $<TARGET_OBJECTS:native_internal>~timer"
+      parameters~$<TARGET_OBJECTS:softBatcher>~cpuBuffer~buffer~\
+      $<TARGET_OBJECTS:native_internal>~timer~memory_pool~batch"
 )
 
 amdinfer_add_unit_tests("${tests}" "${tests_libs}")
diff --git a/tests/unit/batching/test_soft.cpp b/tests/unit/batching/test_soft.cpp
index 906abeb0e..9db61f55d 100644
--- a/tests/unit/batching/test_soft.cpp
+++ b/tests/unit/batching/test_soft.cpp
@@ -15,10 +15,11 @@
 
 #include <memory>  // for allocator
 
-#include "amdinfer/batching/soft.hpp"        // for SoftBatcher
-#include "amdinfer/build_options.hpp"        // for AMDINFER_ENABLE_LOGGING
-#include "amdinfer/core/interface.hpp"       // IWYU pragma: keep
-#include "amdinfer/core/worker_info.hpp"     // for WorkerInfo
+#include "amdinfer/batching/soft.hpp"          // for SoftBatcher
+#include "amdinfer/build_options.hpp"          // for AMDINFER_ENABLE_LOGGING
+#include "amdinfer/core/interface.hpp"         // IWYU pragma: keep
+#include "amdinfer/core/memory_pool/pool.hpp"  // for MemoryPool
+#include "amdinfer/core/worker_info.hpp"       // for WorkerInfo
 #include "amdinfer/observation/logging.hpp"  // for initLogger, LogLevel, Log...
 #include "gtest/gtest.h"                     // for Test, SuiteApiResolver, TEST
 
@@ -38,11 +39,13 @@ TEST(UnitSoftBatcher, ConstructAndStart) {
   initLogger(options);
 #endif
 
-  SoftBatcher batcher;
+  MemoryPool pool;
+
+  SoftBatcher batcher(&pool);
   batcher.setName("test");
 
-  WorkerInfo fake("", nullptr);
-  batcher.start(&fake);
+  WorkerInfo fake("", nullptr, &pool);
+  batcher.start({MemoryAllocators::Cpu});
 
   batcher.enqueue(nullptr);
   batcher.end();
diff --git a/tests/unit/batching/test_soft_batching.cpp b/tests/unit/batching/test_soft_batching.cpp
index d605d1bdf..5c908e893 100644
--- a/tests/unit/batching/test_soft_batching.cpp
+++ b/tests/unit/batching/test_soft_batching.cpp
@@ -24,10 +24,10 @@
 #include <vector>            // for vector
 
 #include "amdinfer/batching/soft.hpp"            // for BatchPtr, SoftBatcher
-#include "amdinfer/buffers/vector_buffer.hpp"    // for VectorBuffer
 #include "amdinfer/build_options.hpp"            // for AMDINFER_ENABLE_LOGGING
 #include "amdinfer/clients/native_internal.hpp"  // for CppNativeApi
 #include "amdinfer/core/data_types.hpp"          // for DataType, DataType::U...
+#include "amdinfer/core/memory_pool/pool.hpp"    // for MemoryPool
 #include "amdinfer/core/parameters.hpp"          // for ParameterMap
 #include "amdinfer/core/predict_api.hpp"         // for InferenceRequest, Req...
 #include "amdinfer/core/worker_info.hpp"         // for WorkerInfo
@@ -80,7 +80,6 @@ class UnitSoftBatcherFixture : public testing::TestWithParam<BatchConfig> {
     const auto& batch_config = GetParam();
     int batch_size = batch_config.batch_size;
 
-    const auto buffer_num = 10;
     const auto data_shape = {1UL, 2UL, 50UL};
     // the product of the data shape < 255 to fit into uint8
     const uint8_t data_size = 100;
@@ -92,25 +91,25 @@ class UnitSoftBatcherFixture : public testing::TestWithParam<BatchConfig> {
     parameters.put("timeout", kTimeoutMs);
     parameters.put("batch_size", batch_size);
 
-    this->batcher_.emplace(&parameters);
+    this->batcher_.emplace(&pool_, &parameters);
     this->batcher_->setName("test");
     this->batcher_->setBatchSize(batch_size);
 
-    this->worker_.emplace("", &parameters);
-    for (size_t i = 0; i < buffer_num; i++) {
-      BufferPtrs vec;
-      vec.emplace_back(std::make_unique<VectorBuffer>(batch_size * data_size,
-                                                      DataType::Uint8));
-      this->worker_->putInputBuffer(std::move(vec));
-    }
-    for (size_t i = 0; i < buffer_num; i++) {
-      BufferPtrs vec;
-      vec.emplace_back(std::make_unique<VectorBuffer>(batch_size * data_size,
-                                                      DataType::Uint8));
-      this->worker_->putOutputBuffer(std::move(vec));
-    }
-
-    this->batcher_->start(&this->worker_.value());
+    this->worker_.emplace("", &parameters, &pool_);
+    // for (size_t i = 0; i < buffer_num; i++) {
+    //   BufferPtrs vec;
+    //   vec.emplace_back(std::make_unique<VectorBuffer>(batch_size * data_size,
+    //                                                   DataType::Uint8));
+    //   this->worker_->putInputBuffer(std::move(vec));
+    // }
+    // for (size_t i = 0; i < buffer_num; i++) {
+    //   BufferPtrs vec;
+    //   vec.emplace_back(std::make_unique<VectorBuffer>(batch_size * data_size,
+    //                                                   DataType::Uint8));
+    //   this->worker_->putOutputBuffer(std::move(vec));
+    // }
+
+    this->batcher_->start({MemoryAllocators::Cpu});
 
     data_.resize(data_size);
     for (uint8_t i = 0; i < data_size; i++) {
@@ -183,6 +182,7 @@ class UnitSoftBatcherFixture : public testing::TestWithParam<BatchConfig> {
   InferenceRequest request_;
   std::optional<WorkerInfo> worker_;
   std::optional<SoftBatcher> batcher_;
+  MemoryPool pool_;
 };
 
 TEST_P(UnitSoftBatcherFixture, BasicBatching) {  // NOLINT
diff --git a/tests/unit/buffers/CMakeLists.txt b/tests/unit/buffers/CMakeLists.txt
index 1eaf46bd9..6df0ebccc 100644
--- a/tests/unit/buffers/CMakeLists.txt
+++ b/tests/unit/buffers/CMakeLists.txt
@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-list(APPEND tests vector_buffer)
+list(APPEND tests cpu)
 
-list(APPEND tests_libs "buffer~data_types~fake_observation~vector_buffer")
+list(APPEND tests_libs "buffer~data_types~fake_observation~cpuBuffer")
 
 amdinfer_add_unit_tests("${tests}" "${tests_libs}")
diff --git a/tests/unit/buffers/test_vector_buffer.cpp b/tests/unit/buffers/test_cpu.cpp
similarity index 67%
rename from tests/unit/buffers/test_vector_buffer.cpp
rename to tests/unit/buffers/test_cpu.cpp
index 6a6f5afd3..042b5f775 100644
--- a/tests/unit/buffers/test_vector_buffer.cpp
+++ b/tests/unit/buffers/test_cpu.cpp
@@ -1,5 +1,4 @@
-// Copyright 2022 Xilinx, Inc.
-// Copyright 2022 Advanced Micro Devices, Inc.
+// Copyright 2023 Advanced Micro Devices, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,18 +16,16 @@
 #include <cstddef>  // for size_t
 #include <memory>   // for allocator
 
-#include "amdinfer/buffers/vector_buffer.hpp"  // for Buffer, VectorBuffer
-#include "amdinfer/core/data_types.hpp"        // for DataType, getSize, oper...
-#include "amdinfer/declarations.hpp"           // for BufferPtrs
-#include "amdinfer/util/queue.hpp"             // for BufferPtrsQueue
-#include "gtest/gtest.h"                       // for EXPECT_EQ, FAIL, UnitTest
+#include "amdinfer/buffers/cpu.hpp"      // for CpuBuffer
+#include "amdinfer/core/data_types.hpp"  // for DataType, getSize, oper...
+#include "amdinfer/declarations.hpp"     // for BufferPtrs
+#include "amdinfer/util/queue.hpp"       // for BufferPtrsQueue
+#include "gtest/gtest.h"                 // for EXPECT_EQ, FAIL, UnitTest
 
 namespace amdinfer {
 
 constexpr auto kBufferSize = 10;
-// constexpr auto kDataType = DataType::Int32;
-// constexpr auto kDataSize = types::getSize(kDataType);
-constexpr auto kTimeout = 1E6;  // 1E6 us = 1 s timeout
+constexpr auto kDataSize = 512;
 
 struct WriteData {
   template <typename T>
@@ -56,7 +53,7 @@ struct ReadData {
 
 class UnitVectorBufferFixture : public testing::TestWithParam<DataType> {
  public:
-  UnitVectorBufferFixture() : buffer_(kBufferSize, GetParam()){};
+  UnitVectorBufferFixture() : buffer_(memory_.data(), MemoryAllocators::Cpu){};
 
  protected:
   void SetUp() override {
@@ -68,7 +65,8 @@ class UnitVectorBufferFixture : public testing::TestWithParam<DataType> {
 
   // void TearDown() override {}
 
-  VectorBuffer buffer_;
+  std::vector<std::byte> memory_{kDataSize};
+  CpuBuffer buffer_;
 };
 
 TEST_P(UnitVectorBufferFixture, TestFixtureState) {  // NOLINT
@@ -78,23 +76,7 @@ TEST_P(UnitVectorBufferFixture, TestFixtureState) {  // NOLINT
   }
 }
 
-TEST_P(UnitVectorBufferFixture, TestAllocate) {  // NOLINT
-  BufferPtrsQueue buffer_queue;
-  const size_t buffer_num = 10;
-  const size_t batch_size = 4;
-
-  VectorBuffer::allocate(&buffer_queue, buffer_num, 1 * batch_size, GetParam());
-
-  for (auto i = 0U; i < buffer_num; i++) {
-    BufferPtrs buffers;
-    if (!buffer_queue.wait_dequeue_timed(buffers, kTimeout)) {
-      FAIL() << "Dequeuing buffers timed out";
-    }
-    EXPECT_EQ(buffers.size(), 1);
-  }
-}
-
-// we exclude STRING as it doesn't have a defined size we can pre-allocate
+// Excluding STRING as it doesn't have a defined size to pre-allocate
 // NOLINTNEXTLINE(cert-err58-cpp)
 const std::array<DataType, 12> kDataTypes{
   amdinfer::DataType::Bool,   amdinfer::DataType::Uint8,
diff --git a/tests/unit/core/CMakeLists.txt b/tests/unit/core/CMakeLists.txt
index 643d5ac90..eaf236543 100644
--- a/tests/unit/core/CMakeLists.txt
+++ b/tests/unit/core/CMakeLists.txt
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+add_subdirectory(memory_pool)
+
 list(APPEND tests inference_request_input parameter_map)
 
 list(APPEND tests_libs "predict_api~parameters" "parameters")
diff --git a/tests/unit/core/memory_pool/CMakeLists.txt b/tests/unit/core/memory_pool/CMakeLists.txt
new file mode 100644
index 000000000..affca5303
--- /dev/null
+++ b/tests/unit/core/memory_pool/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Copyright 2023 Advanced Micro Devices, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+list(APPEND tests cpu_allocator pool)
+
+list(APPEND tests_libs "cpu_allocator" "memory_pool~buffer~cpuBuffer")
+
+amdinfer_add_unit_tests("${tests}" "${tests_libs}")
diff --git a/tests/unit/core/memory_pool/test_cpu_allocator.cpp b/tests/unit/core/memory_pool/test_cpu_allocator.cpp
new file mode 100644
index 000000000..2f3d033b1
--- /dev/null
+++ b/tests/unit/core/memory_pool/test_cpu_allocator.cpp
@@ -0,0 +1,88 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+// #include <array>    // for array
+// #include <cstddef>  // for byte, size_t
+// #include <string>   // for string, basic_string, alloc...
+// #include <vector>   // for vector
+
+#include "amdinfer/core/exceptions.hpp"
+#include "amdinfer/core/memory_pool/cpu_allocator.hpp"  // for CpuSimpl...
+#include "amdinfer/testing/gtest.hpp"  // for AssertionResult,...
+
+namespace amdinfer {
+
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST(UnitCpuAllocator, Basic) {
+  CpuAllocator allocator{sizeof(int)};
+
+  const auto* foo = static_cast<int*>(allocator.get(sizeof(int)));
+
+  const auto* bar = static_cast<int*>(allocator.get(sizeof(int)));
+  ASSERT_NE(foo, bar);
+
+  allocator.put(foo);
+  allocator.put(bar);
+}
+
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST(UnitCpuAllocator, Correctness) {
+  CpuAllocator allocator{sizeof(int) * 4, sizeof(int) * 4};
+
+  const auto* foo = static_cast<int*>(allocator.get(sizeof(int)));
+  const auto* bar = static_cast<int*>(allocator.get(sizeof(int)));
+
+  ASSERT_EQ(foo + 1, bar);
+
+  allocator.put(foo);
+  const auto* baz = static_cast<int*>(allocator.get(sizeof(int)));
+
+  ASSERT_EQ(foo, baz);
+
+  allocator.put(baz);
+  allocator.put(bar);
+
+  const auto* bard = static_cast<int*>(allocator.get(sizeof(int) * 4));
+  ASSERT_EQ(foo, bard);
+}
+
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST(UnitCpuAllocator, ExceedingMax) {
+  CpuAllocator allocator{sizeof(int), sizeof(int)};
+
+  std::ignore = static_cast<int*>(allocator.get(sizeof(int)));
+  // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto, hicpp-avoid-goto)
+  EXPECT_THROW_CHECK(
+    std::ignore = static_cast<int*>(allocator.get(sizeof(int)));
+    , EXPECT_STREQ(e.what(), "Too much requested");, runtime_error);
+}
+
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST(UnitCpuAllocator, BadFree) {
+  CpuAllocator allocator{sizeof(int)};
+
+  const auto* foo = static_cast<int*>(allocator.get(sizeof(int)));
+  const auto* bad_address = foo + 1;
+  // NOLINTNEXTLINE(cppcoreguidelines-avoid-goto, hicpp-avoid-goto)
+  EXPECT_THROW_CHECK(allocator.put(bad_address);
+                     , EXPECT_STREQ(e.what(), "Address not found");
+                     , runtime_error);
+}
+
+}  // namespace amdinfer
diff --git a/tests/unit/core/memory_pool/test_pool.cpp b/tests/unit/core/memory_pool/test_pool.cpp
new file mode 100644
index 000000000..3fd33a374
--- /dev/null
+++ b/tests/unit/core/memory_pool/test_pool.cpp
@@ -0,0 +1,37 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief
+ */
+
+#include <tuple>
+
+#include "amdinfer/core/exceptions.hpp"
+#include "amdinfer/core/memory_pool/pool.hpp"
+#include "amdinfer/testing/gtest.hpp"  // for AssertionResult,...
+
+namespace amdinfer {
+
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST(UnitPool, Basic) {
+  MemoryPool pool;
+
+  auto buffer = pool.get({MemoryAllocators::Cpu}, sizeof(int));
+
+  pool.put(std::move(buffer));
+}
+
+}  // namespace amdinfer
diff --git a/tests/workers/test_echo.cpp b/tests/workers/test_echo.cpp
index df12a73c6..79fad0865 100644
--- a/tests/workers/test_echo.cpp
+++ b/tests/workers/test_echo.cpp
@@ -92,7 +92,7 @@ class EchoParamFixture : public testing::TestWithParam<Params> {
     auto add_id = params.add_id;
     auto multiplier = params.multiplier;
 
-    EXPECT_FALSE(response.isError());
+    ASSERT_FALSE(response.isError());
     EXPECT_EQ(response.getModel(), "echo");
     auto outputs = response.getOutputs();
     EXPECT_EQ(outputs.size(), multiplier);
diff --git a/tests/workers/xmodel/test_resnet50.cpp b/tests/workers/xmodel/test_resnet50.cpp
index 303e298a9..c56ed8e95 100644
--- a/tests/workers/xmodel/test_resnet50.cpp
+++ b/tests/workers/xmodel/test_resnet50.cpp
@@ -74,7 +74,7 @@ void validate(const std::vector<InferenceResponse>& responses) {
 
   for (const auto& response : responses) {
     auto outputs = response.getOutputs();
-    EXPECT_EQ(outputs.size(), 1);
+    ASSERT_EQ(outputs.size(), 1);
     auto top_k = postprocess(outputs[0], k);
     for (auto j = 0U; j < k; ++j) {
       EXPECT_EQ(top_k.at(j), golden.at(j));
diff --git a/tests/workers/xmodel/test_yolov3.cpp b/tests/workers/xmodel/test_yolov3.cpp
index a82549b72..b1b4e98fc 100644
--- a/tests/workers/xmodel/test_yolov3.cpp
+++ b/tests/workers/xmodel/test_yolov3.cpp
@@ -69,7 +69,7 @@ std::vector<InferenceRequest> constructRequests(const Images& images) {
   std::vector<InferenceRequest> requests;
   requests.reserve(images.size());
 
-  const std::initializer_list<uint64_t> shape = {224, 224, 3};
+  const std::initializer_list<uint64_t> shape = {416, 416, 3};
 
   for (const auto& image : images) {
     requests.emplace_back();