Repository: cupy/cupy

Issue:
Dynamic Parallelism 
I am interested in utilizing Dynamic Parallelism in my cupy projects. 
```cpp
extern "C"
{

__global__ void test_kernel_inner()
{
    printf("  Hello inner world!\n");

}

__global__ void test_kernel()
{
    printf("Hello outer world!\n");
    test_kernel_inner<<<2, 1>>>();
}

}
```
Trying to launch the above  cuda code with the following commands

```py
TestKernel = cp.RawKernel(Test, 'test_kernel', ('-rdc=true',))
TestKernel((1,),(1,),())
```

I get the error
```
  File "/home/palmerss/PycharmProjects/TestProject/Test.py", line 23, in <module>
    TestKernel((1,),(1,),())
  File "cupy/core/raw.pyx", line 51, in cupy.core.raw.RawKernel.__call__
  File "cupy/util.pyx", line 55, in cupy.util.memoize.decorator.ret
  File "cupy/core/raw.pyx", line 57, in cupy.core.raw._get_raw_kernel
  File "cupy/core/carray.pxi", line 125, in cupy.core.core.compile_with_cache
  File "cupy/core/carray.pxi", line 166, in cupy.core.core.compile_with_cache
  File "/home/palmerss/anaconda3/envs/TestProject/lib/python3.7/site-packages/cupy/cuda/compiler.py", line 168, in compile_with_cache
    cubin = ls.complete()
  File "cupy/cuda/function.pyx", line 216, in cupy.cuda.function.LinkState.complete
  File "cupy/cuda/function.pyx", line 217, in cupy.cuda.function.LinkState.complete
  File "cupy/cuda/driver.pyx", line 161, in cupy.cuda.driver.linkComplete
  File "cupy/cuda/driver.pyx", line 82, in cupy.cuda.driver.check_status
cupy.cuda.driver.CUDADriverError: CUDA_ERROR_UNKNOWN: unknown error
```


Discussion:
ref: https://stackoverflow.com/questions/57152396/dynamic-parallelism-in-cupy

Resolving patch:
diff --git a/cupy/cuda/compiler.py b/cupy/cuda/compiler.py
--- a/cupy/cuda/compiler.py
+++ b/cupy/cuda/compiler.py
@@ -16,6 +16,10 @@
 
 _nvrtc_version = None
 _nvrtc_max_compute_capability = None
+_win32 = sys.platform.startswith('win32')
+_rdc_flags = ('--device-c', '-dc', '-rdc=true',
+              '--relocatable-device-code=true')
+_cudadevrt = None
 
 
 class NVCCException(Exception):
@@ -67,6 +71,39 @@ def _get_arch():
                _nvrtc_max_compute_capability)
 
 
+def _is_cudadevrt_needed(options):
+    return any(o for o in options if o in _rdc_flags)
+
+
+def _get_cudadevrt_path():
+    # defer import to here to avoid circular dependency
+    from cupy.cuda import get_cuda_path
+    global _win32
+
+    cudadevrt = get_cuda_path()
+    if cudadevrt is None:
+        raise RuntimeError('CUDA is not found.')
+
+    if _win32:
+        # rely on os.altsep
+        cudadevrt += '/lib/x64/cudadevrt.lib'
+    else:  # linux & osx: search twice as in cupy/install/build.py
+        cudadevrt64 = cudadevrt + '/lib64/libcudadevrt.a'
+        if not os.path.isfile(cudadevrt64):
+            cudadevrt += '/lib/libcudadevrt.a'
+        else:
+            cudadevrt = cudadevrt64
+    if not os.path.isfile(cudadevrt):
+        raise RuntimeError(
+            'Relocatable PTX code is requested, but cudadevrt '
+            'is not found.')
+    return cudadevrt
+
+
+def _remove_rdc_option(options):
+    return tuple(o for o in options if o not in _rdc_flags)
+
+
 class TemporaryDirectory(object):
     def __enter__(self):
         self.path = tempfile.mkdtemp()
@@ -117,15 +154,18 @@ def compile_using_nvrtc(source, options=(), arch=None, filename='kern.cu'):
 
 
 def compile_using_nvcc(source, options=(), arch=None,
-                       filename='kern.cu', code_type='cubin'):
+                       filename='kern.cu', code_type='cubin',
+                       separate_compilation=False):
     if not arch:
         arch = _get_arch()
 
     if code_type not in ('cubin', 'ptx'):
         raise ValueError('Invalid code_type %s. Should be cubin or ptx')
+    if code_type == 'ptx':
+        assert not separate_compilation
 
     arch_str = '-gencode=arch=compute_{cc},code=sm_{cc}'.format(cc=arch)
-    cmd = ['nvcc', '--%s' % code_type, arch_str] + list(options)
+    cmd = ['nvcc', arch_str]
 
     with TemporaryDirectory() as root_dir:
         first_part = filename.split('.')[0]
@@ -137,19 +177,53 @@ def compile_using_nvcc(source, options=(), arch=None,
         with open(cu_path, 'w') as cu_file:
             cu_file.write(source)
 
-        cmd.append(cu_path)
-
-        try:
-            _run_nvcc(cmd, root_dir)
-        except NVCCException as e:
-            cex = CompileException(str(e), source, cu_path, options, 'nvcc')
-
-            dump = _get_bool_env_variable(
-                'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
-            if dump:
-                cex.dump(sys.stderr)
-
-            raise cex
+        if not separate_compilation:  # majority cases
+            cmd.append('--%s' % code_type)
+            cmd += list(options)
+            cmd.append(cu_path)
+
+            try:
+                _run_nvcc(cmd, root_dir)
+            except NVCCException as e:
+                cex = CompileException(str(e), source, cu_path, options,
+                                       'nvcc')
+
+                dump = _get_bool_env_variable(
+                    'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+                if dump:
+                    cex.dump(sys.stderr)
+
+                raise cex
+        else:  # two steps: compile to object and device-link
+            cmd_partial = cmd.copy()
+            cmd_partial.append('--cubin')
+
+            obj = path + '.o'
+            cmd += list(options + ('-o', obj))
+            cmd.append(cu_path)
+
+            try:
+                _run_nvcc(cmd, root_dir)
+            except NVCCException as e:
+                cex = CompileException(str(e), source, cu_path, options,
+                                       'nvcc')
+
+                dump = _get_bool_env_variable(
+                    'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+                if dump:
+                    cex.dump(sys.stderr)
+
+                raise cex
+
+            options = _remove_rdc_option(options)
+            options += ('--device-link', obj, '-o', path + '.cubin')
+            cmd = cmd_partial + list(options)
+
+            try:
+                _run_nvcc(cmd, root_dir)
+            except NVCCException as e:
+                cex = CompileException(str(e), '', '', options, 'nvcc')
+                raise cex
 
         if code_type == 'ptx':
             with open(result_path, 'rb') as ptx_file:
@@ -263,10 +337,17 @@ def _compile_with_cache_cuda(source, options, arch, cache_dir,
         ptx = compile_using_nvrtc(source, options, arch, name + '.cu')
         ls = function.LinkState()
         ls.add_ptr_data(ptx, 'cupy.ptx')
+        # for separate compilation
+        if _is_cudadevrt_needed(options):
+            global _cudadevrt
+            if _cudadevrt is None:
+                _cudadevrt = _get_cudadevrt_path()
+            ls.add_ptr_file(_cudadevrt)
         cubin = ls.complete()
     elif backend == 'nvcc':
+        rdc = _is_cudadevrt_needed(options)
         cubin = compile_using_nvcc(source, options, arch, name + '.cu',
-                                   code_type='cubin')
+                                   code_type='cubin', separate_compilation=rdc)
     else:
         raise ValueError('Invalid backend %s' % backend)
 


Test patch:
diff --git a/tests/cupy_tests/core_tests/test_raw.py b/tests/cupy_tests/core_tests/test_raw.py
--- a/tests/cupy_tests/core_tests/test_raw.py
+++ b/tests/cupy_tests/core_tests/test_raw.py
@@ -83,6 +83,29 @@
 }
 '''
 
+# dynamic parallelism
+_test_source4 = r'''
+extern "C"{
+
+__global__ void test_kernel_inner(float *arr, int N)
+{
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if (tid < N)
+        arr[tid] = 1.0;
+}
+
+__global__ void test_kernel(float *arr, int N, int inner_blk)
+{
+    unsigned int tid = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if (tid < N/inner_blk)
+        test_kernel_inner<<<1, inner_blk>>>(arr+tid*inner_blk, inner_blk);
+}
+
+}
+'''
+
 _test_cuComplex = r'''
 #include <cuComplex.h>
 #define N 100
@@ -341,9 +364,33 @@ def test_get_function_failure(self):
         # this error is more likely to appear when using RawModule, so
         # let us do it here
         with pytest.raises(cupy.cuda.driver.CUDADriverError) as ex:
-            self.mod2.get_function("no_such_kernel")
+            self.mod2.get_function('no_such_kernel')
         assert 'CUDA_ERROR_NOT_FOUND' in str(ex.value)
 
+    def test_dynamical_parallelism(self):
+        ker = cupy.RawKernel(_test_source4, 'test_kernel', options=('-dc',),
+                             backend=self.backend)
+        N = 169
+        inner_chunk = 13
+        x = cupy.zeros((N,), dtype=cupy.float32)
+        ker((1,), (N//inner_chunk,), (x, N, inner_chunk))
+        assert (x == 1.0).all()
+
+    def test_dynamical_parallelism_compile_failure(self):
+        # no option for separate compilation is given should cause an error
+        ker = cupy.RawKernel(_test_source4, 'test_kernel',
+                             backend=self.backend)
+        N = 10
+        inner_chunk = 2
+        x = cupy.zeros((N,), dtype=cupy.float32)
+        if self.backend == 'nvrtc':
+            # raised when calling ls.complete()
+            with pytest.raises(cupy.cuda.driver.CUDADriverError):
+                ker((1,), (N//inner_chunk,), (x, N, inner_chunk))
+        else:  # nvcc
+            with pytest.raises(cupy.cuda.compiler.CompileException):
+                ker((1,), (N//inner_chunk,), (x, N, inner_chunk))
+
     def test_cuFloatComplex(self):
         N = 100
         block = 32