Torchlib C++ GPU memory is not released after inference function is called

JunaidAwan: <p>I am trying to build torchlib c++ shared library using cmake for using it later in python. My c++ class is as follows:</p>
<pre><code class="lang-auto">#include &lt;torch/torch.h&gt;
#include &lt;torch/script.h&gt;
// This header is what defines the custom class registration
// behavior specifically. script.h already includes this, but
// we include it here so you know it exists in case you want
// to look at the API or implementation.
#include &lt;torch/custom_class.h&gt;

#include &lt;string&gt;
#include &lt;vector&gt;
#include &lt;ATen/ATen.h&gt;
#include &lt;cuda_runtime_api.h&gt;
#include &lt;cuda.h&gt;

#include &lt;c10/cuda/CUDACachingAllocator.h&gt;

template &lt;class T&gt;
struct MyStackClass : torch::CustomClassHolder {
  std::vector&lt;T&gt; stack_;
  torch::jit::script::Module module;
  
  MyStackClass(std::string init) {
    try {
        module = torch::jit::load(init, torch::kCUDA);
        module.eval();
        
      }
      catch (const c10::Error&amp; e) {
        std::cerr &lt;&lt; "error loading the face model\n";
      }
  }
  
  /* function definition */  
  at::Tensor get_face_feat(torch::Tensor&amp; input_tensor) {
      std::vector&lt;torch::jit::IValue&gt; inputs;
      inputs.push_back(input_tensor); //.to(at::kCUDA));
      // Execute the model and turn its output into a tensor.
      at::Tensor output = module.forward(inputs).toTensor();
      output = output.to(at::kCPU);
      c10::cuda::CUDACachingAllocator::emptyCache();
      
      
      return output;
  }


  
};


TORCH_LIBRARY(my_classes, m) {
  m.class_&lt;MyStackClass&lt;std::string&gt;&gt;("MyStackClass")
    .def(torch::init&lt;std::string&gt;())
    .def("get_face_feat", &amp;MyStackClass&lt;std::string&gt;::get_face_feat)
    
    
  ;
}
</code></pre>
<p>From cmake I can build the library</p>
<pre><code class="lang-auto">cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
project(custom_class)

find_package(Torch REQUIRED )

find_package( CUDA REQUIRED)

# Define our library target
add_library(custom_class SHARED class.cpp)
set(CMAKE_CXX_STANDARD 14)
# Link against LibTorch
target_link_libraries(custom_class "${TORCH_LIBRARIES}")
</code></pre>
<p>In python I can use the library as follows:</p>
<pre><code class="lang-auto">import torch
print(torch.__version__)
import cv2
from torchvision import transforms
import sys
# `torch.classes.load_library()` allows you to pass the path to your .so file
# to load it in and make the custom C++ classes available to both Python and
# TorchScript
torch.classes.load_library("build/libcustom_class.so")
# You can query the loaded libraries like this:
print(torch.classes.loaded_libraries)
s = torch.classes.my_classes.MyStackClass("&lt;path_to_checkpoint.pth&gt;")
im = cv2.imread('&lt;path_to_image.jpg&gt;')

transform = transforms.Compose([ ...  ])
normface = cv2.resize(im, (112,112))
normface = transform(normface)
normface = torch.unsqueeze(normface, 0)

feat = s.get_face_feat(normface.to('cuda'))  

</code></pre>
<p>Now the problem is that GPU memory is not being released after the last line <strong>feat = s.get_face_feat(normface.to(‘cuda’))</strong>  and gpu memory keeps on increasing on each call. Any guidance is appreciated.</p>