Repository: pytorch
Issue: Difference in inference time between CUDA 10.0 & 10.2
Labels: module: performance,module: cudnn,module: cuda,triaged

## 🐛 Bug

Running the same model using LibTorch that was compiled with CUDA 10.2 results in ~20x slower inference time than using LibTorch that was compiled with CUDA 10.0

## To Reproduce

Steps to reproduce the behavior:

Compile LibTorch using 10.0 & 10.2
Run the following code with both libtorch libraries

```
#include "TorchTester.h"

// STL
#include <memory>
#include <iostream>
#include <string>
#include <cassert>

// Torch
#include <torch/script.h>
#include <c10/cuda/CUDAStream.h>
#include <c10/cuda/CUDAGuard.h>

// CUDA
#include <cuda_runtime_api.h>

#define DEFAULT_TORCH_SCRIPT ""
#define PATCH_WIDTH (256)
#define PATCH_HEIGHT (256)


using JitModule = torch::jit::Module;

bool prepareInput(const uint8_t* inputData, int width, int height, std::vector<torch::jit::IValue>& inputTensor, at::cuda::CUDAStream& stream)
{
	torch::Tensor tensorImageGpu;

	std::vector<int64_t> dims = { 1, height, width, 3 };
	auto options = torch::TensorOptions().dtype(torch::kUInt8).device({ torch::kCUDA }).requires_grad(false);
	tensorImageGpu = torch::zeros(dims, options);

	auto pitch = width * sizeof(uint8_t) * 3;
	auto data = tensorImageGpu.data<uint8_t>();
	cudaErr = cudaMemcpy2DAsync(data,
		pitch,
		inputData,
		pitch,
		pitch,
		height,
		cudaMemcpyHostToDevice,
		stream.stream());
	if (cudaErr != cudaSuccess)
	{
		std::cerr << "Error copying data" << std::endl;
		return false;
	}
	cudaStreamSynchronize(stream.stream());

	tensorImageGpu = tensorImageGpu.to(torch::kFloat);

	tensorImageGpu = tensorImageGpu / 255.0;

	// Transpose the image for [channels, rows, columns] format of Pytorch tensor
	tensorImageGpu = at::transpose(tensorImageGpu, 1, 2);
	tensorImageGpu = at::transpose(tensorImageGpu, 1, 3);

	if (!inputTensor.empty())
	{
		inputTensor.clear();
	}
	inputTensor.push_back(tensorImageGpu);

	return true;
}


bool infer(std::shared_ptr<JitModule>& jitModule, at::cuda::CUDAStream& stream, const uint8_t* inputData, int width, int height)
{
	std::vector<torch::jit::IValue> tensorInput;
	if (!prepareInput(inputData, width, height, tensorInput, stream))
	{
		return false;
	}
	// Reduce memory usage, without gradients
	torch::NoGradGuard noGrad;
	{
		at::cuda::CUDAStreamGuard streamGuard(stream);
		
		auto totalTimeStart = std::chrono::high_resolution_clock::now();

		jitModule->forward(tensorInput);
		// This synchronize is just for timing, not used in production
		cudaStreamSynchronize(stream.stream());
		
		auto totalTimeStop = std::chrono::high_resolution_clock::now();
		printf("forward sync time = %.3f milliseconds\n",
			std::chrono::duration<double, std::milli>(totalTimeStop - totalTimeStart).count());
	}

	return true;
}

int main(int argc, char** argv)
{
	
	// Initialize CUDA device 0
	cudaSetDevice(0);

	std::string networkPath = DEFAULT_TORCH_SCRIPT; // Torch script that has an input of 256x256x3
	if (argc > 1)
	{
		networkPath = argv[1];
	}
	auto jitModule = std::make_shared<torch::jit::Module>(torch::jit::load(networkPath, torch::kCUDA));
	if (jitModule == nullptr)
	{
		std::cerr << "Failed creating network" << std::endl;
		return EXIT_FAILURE;
	}

	uint8_t* data = new uint8_t[PATCH_HEIGHT * PATCH_WIDTH * 3];
	memset(data, 0, PATCH_HEIGHT * PATCH_WIDTH * 3);
	
	auto stream = at::cuda::getStreamFromPool(true, 0);

	bool res = infer(jitModule, stream, data, PATCH_WIDTH, PATCH_HEIGHT);

	std::cout << "Warmed up" << std::endl;


	res = infer(jitModule, stream, data, PATCH_WIDTH, PATCH_HEIGHT);

	delete[] data;
	return 0;
}
```

## Expected behavior

Runtime of both runs should be roughly the same

## Environment

 - PyTorch Version (e.g., 1.0): 1.5.0
 - OS (e.g., Linux): Windows 10 & Ubuntu 16.04
 - How you installed PyTorch (`conda`, `pip`, source): Compiled from sources & tested precompiled binaries
 - Build command you used (if compiling from source):
```
USE_OPENCV=1 \
USE_CUDA=1 \
USE_NNPACK=1 \
TORCH_CUDA_ARCH_LIST="5.2 6.0 6.1 7.0 7.5+PTX" \
TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \
python3 setup.py install
```
 - Python version: 3.7
 - CUDA/cuDNN version: CUDA 10.0 with cuDNN 7.6.3  & CUDA 10.2 with cuDNN 7.6.5
 - GPU models and configuration: Tested on GTX 980, T4 & P1000
 - Any other relevant information:

Question on StackOverflow:
[https://stackoverflow.com/q/63035907/975959](url)


cc @ngimel @csarofeen @ptrblck @VitalyFedyunin