OpenGL, Libtorch and cuda interop. Doing inference on texture data

username: emiven
cooked: <p>So i’m running an en/decoder network for image colorization on Opengl texture data.<br>
wanting to eek out as much performance as possible i want to keep all the data on the GPU so I’m using CUDA/OpenGL interop to map the texture as graphics resource, create a CUDA array with it using cudaGraphicsSubResourceGetMappedArray, copying the data to a device pointer that i use torch::from_blob() to create a torch tensor with.<br>
then if i just copy this tensor to another CUDA array wich i’ve mapped another texture to, i get a exact copy of the original image back to the new rendered texture.</p>
<p>The problem is that i’m not sure how to get the RGBA texture data into a (c,h,w) format that the network can use, and then convert to a format that OpengGl may use again. currently the model expects a 1,1,299,299 tensor (the L in CIELAB color space) and outputs a (2,299,299) tensor (the AB channels). The tensor must then be concatenated before copied to the texture mapped CUDA array. This is the current output when running the texture data through the network: (299x299 texture in left corner)<br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/f/b/fb3a28e2bfd6821a5728fec03283296214b72a3d.png" data-download-href="https://discuss.pytorch.org/uploads/default/fb3a28e2bfd6821a5728fec03283296214b72a3d" title="image"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/f/b/fb3a28e2bfd6821a5728fec03283296214b72a3d_2_329x250.png" alt="image" data-base62-sha1="zQss0hy3eOZ8AlZx0a8MMcV8vsF" width="329" height="250" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/f/b/fb3a28e2bfd6821a5728fec03283296214b72a3d_2_329x250.png, https://discuss.pytorch.org/uploads/default/optimized/3X/f/b/fb3a28e2bfd6821a5728fec03283296214b72a3d_2_493x375.png 1.5x, https://discuss.pytorch.org/uploads/default/optimized/3X/f/b/fb3a28e2bfd6821a5728fec03283296214b72a3d_2_658x500.png 2x" data-dominant-color="101010"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">1178×893 31.3 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>
<p>replacing shading with flat white color:</p>
<p><img src="https://discuss.pytorch.org/uploads/default/original/3X/2/e/2e2d86ca763b61bf590941dfd3368d7c99e194b6.png" alt="image" data-base62-sha1="6AvvevzCUo9mClfFM8WWMAj8hx4" width="254" height="238"></p>
<p>here are some snippets of code:</p>
<pre><code class="lang-auto">// init
	CHECK_CUDA_ERROR(cudaGraphicsGLRegisterImage(&amp;m_imageTextureResource, m_imageTexture-&gt;id(), GL_TEXTURE_2D, cudaGraphicsRegisterFlagsNone));
	CHECK_CUDA_ERROR(cudaGraphicsGLRegisterImage(&amp;m_colorTextureResource, m_colorTexture-&gt;id(), GL_TEXTURE_2D, cudaGraphicsRegisterFlagsNone));
//.....
//.....

//rendering
//.....
// map resource for use with cuda
	CHECK_CUDA_ERROR(cudaGraphicsMapResources(1, &amp;m_imageTextureResource));
	CHECK_CUDA_ERROR(cudaGraphicsMapResources(1, &amp;m_colorTextureResource));
	// get cuda array from mapped resource
	cudaArray_t imageDeviceArray;
	CHECK_CUDA_ERROR(cudaGraphicsSubResourceGetMappedArray(&amp;imageDeviceArray, m_imageTextureResource, 0, 0));

	cudaArray_t colorDeviceArray;
	CHECK_CUDA_ERROR(cudaGraphicsSubResourceGetMappedArray(&amp;colorDeviceArray, m_colorTextureResource, 0, 0));
	cudaChannelFormatDesc desc;
	cudaExtent extent;
	uint flags;
	CHECK_CUDA_ERROR(cudaArrayGetInfo(&amp;desc, &amp;extent, &amp;flags, imageDeviceArray));

	// to be able to pass the data to torch, we need to copy the cuda array to a device pointer
	// here, we allocate the necessary memory
	void* devicePtr = nullptr;
	CHECK_CUDA_ERROR(cudaMalloc(&amp;devicePtr, extent.width* extent.height * 4 * sizeof(float)));

	// now we copy the array to the device pointer
	CHECK_CUDA_ERROR(cudaMemcpy2DFromArray(devicePtr, extent.width * 4 * sizeof(float), imageDeviceArray, 0, 0, extent.width * 4 * sizeof(float), extent.height, cudaMemcpyDeviceToDevice));
	{
int64_t(extent.width) };
		//auto tensor = torch::from_blob(devicePtr,{1,3,long long(extent.height) ,long long(extent.width) }, torch::TensorOptions().dtype(torch::kFloat32).layout(torch::kStrided).device(torch::kCUDA));
		
		auto texture_tensor = torch::from_blob(devicePtr, { 1, long long(extent.height), long long(extent.width), 3 }, torch::TensorOptions().dtype(torch::kF32).layout(torch::kStrided).device(torch::kCUDA));
		texture_tensor = texture_tensor.permute({ 0,3,1,2 });
		//auto test = torch::ones(torch::IntArrayRef{ int64_t(1), int64_t(3), int64_t(m_imageSize.y), int64_t(m_imageSize.x) }, torch::TensorOptions().dtype(torch::kFloat32).layout(torch::kStrided).device(torch::kCUDA));

		try {

			std::vector&lt;torch::jit::IValue&gt; inputs;
			inputs.push_back(texture_tensor);
			auto embedding = m_ModelInceptionV3.forwardPass(inputs);

			inputs.clear();
			auto encoderL = texture_tensor.index({ 0,0 }).unsqueeze(0).unsqueeze(0);
			std::cout &lt;&lt; "encoder L: " &lt;&lt; encoderL.sizes() &lt;&lt; std::endl;
			inputs.push_back(encoderL);
			inputs.push_back(embedding);

			auto EncoderAB = m_ModelEncoder.forwardPass(inputs);

			auto test = torch::cat({ encoderL.squeeze(0) ,EncoderAB.squeeze()}, 0);

			cudaMemcpy2DToArray(colorDeviceArray, 0, 0, test.data_ptr&lt;float&gt;(), m_imageSize.x * 4 * sizeof(float), m_imageSize.x * 4 * sizeof(float), m_imageSize.y, cudaMemcpyDeviceToDevice);
		}
		catch (const c10::Error&amp; e) {
			std::cerr &lt;&lt; "inference error\n" &lt;&lt; e.what() &lt;&lt; std::endl;
		}
	
	}

	CHECK_CUDA_ERROR(cudaFree(devicePtr));
	devicePtr = nullptr;

	// unmap resource (to be able to use it in OpenGL again)
	CHECK_CUDA_ERROR(cudaGraphicsUnmapResources(1, &amp;m_imageTextureResource));
	CHECK_CUDA_ERROR(cudaGraphicsUnmapResources(1, &amp;m_colorTextureResource));
</code></pre>

username: rbaleksandar
cooked: <p>Did you manage to solve the problem?</p>