Repository: pytorch
Issue: libtorch memory leakage when inference by Multi-thread
Labels: module: cpp,module: memory usage,triaged

### 🐛 Describe the bug

When I use libtorch2.1.0 (I have tried other versions), there is a memory leak in the following code, which will increase steadily by 01.~0.2MB each time. I tried to use pointers to release memory, but it did not solve this problem.After positioning, I found that it was the following line of code：
```c++
	*mOutput = m_pNet->m_model.forward(vInputs).toTensor();
```
At the same time, deleting these two line of code is likely to cause program crash：
```c++
	torch::set_num_threads(1);
	torch::NoGradGuard no_grad;
```
Part of my code is as follows：

```c++
bool ClassifierCNN::InitModel(const std::string& strNetPath, const cv::Size& mSize, const std::vector<std::string>& vClassName, bool bCuda)
{

	torch::set_num_threads(1);
	// at::set_num_threads(1);


	std::cout << bCuda << "," << torch::cuda::is_available() << std::endl;
	// cpu or gpu
	if (torch::cuda::is_available())
	{
		std::cout << "CUDA is available" << std::endl;
		if (bCuda)
		{
			m_pNet->m_deviceType = torch::kCUDA;
			std::cout << "Device: CUDA" << std::endl;
		}
		else
		{
			m_pNet->m_deviceType = torch::kCPU;
			std::cout << "Device: CPU" << std::endl;
		}
	}
	else
	{
		m_pNet->m_deviceType = torch::kCPU;
		std::cout << "Device: CPU" << std::endl;
	}


	try {
		m_pNet->m_model = torch::jit::load(strNetPath);
	}
	catch (const c10::Error& e)
	{
		std::cerr << "Error loading the model" << std::endl;
		return false;
	}
	//std::cout << "load sucessful" << std::endl;

	torch::Device mDevice(m_pNet->m_deviceType);
	m_pNet->m_model.to(mDevice);
	m_pNet->m_model.eval();

	m_inputSize = mSize;
	m_vClassName = vClassName;
	WarmUp();
	//std::cout << "WarmUp sucessful" << std::endl;

	return true;
}

bool ClassifierCNN::PredictImage(const cv::Mat& mSrcImg, std::string& strRstName)
{
	if (CV_8UC3 != mSrcImg.type() || 0 == m_vClassName.size())
	{
		return false;
	}
	 cv::Mat* mPredictImg = new cv::Mat;

	if ((mSrcImg.cols != m_inputSize.width) & (mSrcImg.rows != m_inputSize.height))
	{
		cv::resize(mSrcImg, *mPredictImg, m_inputSize);
	}
	delete mPredictImg;
	mPredictImg = nullptr;
	torch::NoGradGuard no_grad;
	// mPredictImg.convertTo(mPredictImg, CV_32FC3, 1. / 255.);
	// cv::subtract(mPredictImg, cv::Scalar(0.406, 0.456, 0.485), mPredictImg);
	// cv::divide(mPredictImg, cv::Scalar(0.225, 0.224, 0.229), mPredictImg);
	// cv::cvtColor(mPredictImg, mPredictImg, cv::COLOR_BGR2RGB);
	torch::Tensor mInputTensor = torch::from_blob(mSrcImg.data, { 1, m_inputSize.height, m_inputSize.width, 3 });
	mInputTensor = mInputTensor.permute({ 0, 3, 1, 2 });
	//std::cout << "mInputTensor sucessful" << std::endl;
	std::vector<torch::jit::IValue> vInputs;
	vInputs.push_back(mInputTensor.to(m_pNet->m_deviceType));
	std::cout << "vInputs sucessful" << std::endl;
	at::Tensor* mOutput = new at::Tensor;
	*mOutput = m_pNet->m_model.forward(vInputs).toTensor();
	delete mOutput;
	mOutput = nullptr;
	return true;}
```

```C++
while (true)
        {
            std::thread thread([&]
                {
                net.PredictImage(mSrcImg, strRstName, fScore);
                });
            thread.join();
            std::this_thread::sleep_for(std::chrono::milliseconds(2000));
        }
```
### Versions

VS2019
Libtorch2.1.0
C++17
opencv4.7.0
```[tasklist]
### Tasks
- [ ] https://github.com/pytorch/pytorch/issues/24237
- [ ] https://github.com/pytorch/pytorch/issues/61920
- [ ] https://github.com/pytorch/pytorch/issues/64412
- [ ] https://github.com/pytorch/pytorch/issues/26165
```

cc @jbschlosser