Issue 1:
I am using torch1.5 + cuda10, call cuda-function with c++ code and use it in python by pybind.

1、paste my c++ code, name bind_cudevice.cc

2、compile bind_cudevice.cc with my g++ compile code

3、paste my python code named test.py

4、Finally you will see core dump, this core only with cuda10+torch1.5

My Environment

PyTorch Version (e.g., 1.0):1.5

OS (e.g., Linux):Linux

How you installed PyTorch (conda, pip, source):source

Build command you used (if compiling from source):yes

Python version:3.6.8

CUDA/cuDNN version:Cuda10.0

GPU models and configuration:K40

Any other relevant information:

The c++ code as below:

#include <iostream>
#include <cublas_v2.h>
#include <cuda.h>
#include <cuda_runtime_api.h>
#include <cusparse.h>
#include <cusolverDn.h>
#include <curand.h>
#include <pybind11/pybind11.h>

class CuDevice {
public:
static CuDevice& Instantiate() {
CuDevice &ans = this_thread_device_;
if (!ans.initialized_)
ans.Initialize();
return ans;
}

private:
CuDevice(){};
public:
void Initialize() {
initialized_ = true;
cusolverDnCreate(&cusolverdn_handle_);
cusolverDnSetStream(cusolverdn_handle_, cudaStreamPerThread);
}
~CuDevice() {
std::cout << "in ~CuDevice" << std::endl;
if (cusolverdn_handle_) {
std::cout << "in ~CuDevice and cusolverdn_handle_" << std::endl;
cusolverDnDestroy(cusolverdn_handle_);
}
}
private:
bool initialized_;
cublasHandle_t cublas_handle_;
cusparseHandle_t cusparse_handle_;
curandGenerator_t curand_handle_;
cusolverDnHandle_t cusolverdn_handle_;
static thread_local CuDevice this_thread_device_;
//static CuDevice this_thread_device_;
};

thread_local CuDevice CuDevice::this_thread_device_ = CuDevice();

int init_cudevice(){
CuDevice::Instantiate();
std::cout << "hello world" << std::endl;
return 0;
}

PYBIND11_MODULE(example, m) {
m.def("init_cudevide",
&init_cudevice,
"select gpu id");
}

Issue 2:
🐛 Bug

I am using cusolverDn in c++ and import it to python with pybind11.

Segment Fault after use cusolverDnCreate() and cusolverDnDestroy() with torch.

NOTE:

###1: When the demo run with Torch1.2,it run success, the SegFault disappear, does it mean Torch 1.5 has some bug?

###2: This SegFault only appear with "thread_local", Does it provide some information for this bug?

###3: Please see test.py below, if we "import torch" after "import example", SegFault disappear.

###4: Torch have not link libcusolver.so, so we just link it once.

@ezyang say that upgarde cuda may fix this, but after i upgrade cuda to 10.2, SegFault as usual.

cc please @ezyang @gchanan @zou3519 @yf225 @glaringlee @ngimel @mcarilli

To Reproduce

Steps to reproduce the behavior:

1.paste the c++ code, name bind_cudevice.cc

2.compile bind_cudevice.cc with my g++ compile code

3.paste my python code named test.py

4. just run test.py and see

bind_cudevice.cc

#include <iostream>

#include <cublas_v2.h>
#include <cuda.h>
#include <cuda_runtime_api.h>

#include <cusolverDn.h>
#include <cusolverSp.h>
#include <cusolverRf.h>

#include <pybind11/pybind11.h>

class CuDevice {

public:
CuDevice() {
initialized_ = true;

cudaSetDevice(0);

cusolverDnCreate(&cusolverdn_handle_);
cusolverDnSetStream(cusolverdn_handle_, cudaStreamPerThread);
}

~CuDevice() {
std::cout << "in ~CuDevice" << std::endl;
if (cusolverdn_handle_) {
std::cout << "in ~CuDevice and cusolverdn_handle_" << std::endl;
cusolverDnDestroy(cusolverdn_handle_);
std::cout << "Destory success" << std::endl;
}
}

private:
bool initialized_;
cusolverDnHandle_t cusolverdn_handle_;

};

int init_cudevice(){
thread_local CuDevice a = CuDevice();
std::cout << "hello world" << std::endl;
return 0;
}

PYBIND11_MODULE(example, m) {
m.def("init_cudevide",
&init_cudevice,
"select gpu id");
}

g++ compile code

g++ -O3 -Wall -shared -std=c++11 -fPIC -I/usr/include/python3.6m -I./third-party/pybind11-2.2.4/include -I/usr/local/cuda/include -Wl,-rpath=/usr/local/cuda/lib64 -L/usr/local/cuda/lib64 -lcudart -lcusolver `python3 -m pybind11 --includes` binding_cudevice.cc -o example`python3-config --extension-suffix`

test.py

import torch
import example
example.init_cudevide()

Core Dump Info

(gdb) bt
#0 0x00007f78b6a391de in ?? () from /usr/local/cuda/lib64/libcusolver.so.10.0
#1 0x00007f78b6a5b4ca in ?? () from /usr/local/cuda/lib64/libcusolver.so.10.0
#2 0x00007f78b5db6e49 in ?? () from /usr/local/cuda/lib64/libcusolver.so.10.0
#3 0x00007f78b5b854c1 in cusolverDnDestroy () from /usr/local/cuda/lib64/libcusolver.so.10.0
#4 0x00007f78be143952 in kaldi::CuDevice::~CuDevice (this=0x2f18a38, __in_chrg=<optimized out>) at cu-device.cc:683
#5 0x00007f79880d7ad1 in (anonymous namespace)::run (p=<optimized out>) at /home/nwani/m3/conda-bld/compilers_linux-64_1560109574129/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/libsupc++/atexit_thread.cc:75
#6 0x00007f79b752cc99 in __run_exit_handlers () from /usr/lib64/libc.so.6
#7 0x00007f79b752cce7 in exit () from /usr/lib64/libc.so.6
#8 0x00007f79b751550c in __libc_start_main () from /usr/lib64/libc.so.6
#9 0x0000000000400c20 in _start ()

Expected behavior

No error happen

Environment

PyTorch Version (e.g., 1.5):1.5

OS (e.g., Linux):Centos7

How you installed PyTorch (conda, pip, source):source

Build command you used (if compiling from source):

Python version:3.6.8

CUDA/cuDNN version:10.0

GPU models and configuration:K40

Any other relevant information:

Additional context

cuSolver is base cublas and cusparse, reference about cuSolver：

https://docs.nvidia.com/cuda/archive/10.0/cusolver/index.html#cuSolverDNhandle

cc @ngimel