Repository: cupy/cupy

Issue:
ROCm 3.5.0 test: hiprtc does not work
**(UPDATED)** I am testing #3238 which just got merged, and looks like the "-I" compiler flag is not picked up by hiprtc, so it complains CuPy's headers not found. This is not CuPy's problem, so we will need to disable it for the time being. (See my reply below.)

By setting two env vars `AMD_COMGR_EMIT_VERBOSE_LOGS=1` and `AMD_COMGR_REDIRECT_LOGS=stdout` for comgr and also `CUPY_DUMP_CUDA_SOURCE_ON_ERROR=1` for cupy, this is the full log I got:
```
$ pytest tests/cupy_tests/core_tests/test_raw.py -x -rA
========================================================================= test session starts =========================================================================
platform linux -- Python 3.7.8, pytest-6.0.1, py-1.9.0, pluggy-0.13.1
rootdir: /home/leo/dev/cupy_rocm35, configfile: setup.cfg
collected 112 items                                                                                                                                                   

tests/cupy_tests/core_tests/test_raw.py F

============================================================================== FAILURES ===============================================================================
____________________________________________________ TestRaw_param_0_{backend='nvrtc', in_memory=False}.test_basic ____________________________________________________

self = <cupy.cuda.compiler._NVRTCProgram object at 0x7f9ef22db050>
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include', '-arch=compute_gfx906', '-I', '/home/leo/dev/cupy_rocm35/cupy/core/include'), log_stream = None

    def compile(self, options=(), log_stream=None):
        try:
            if self.name_expressions:
                for ker in self.name_expressions:
                    nvrtc.addAddNameExpression(self.ptr, ker)
>           nvrtc.compileProgram(self.ptr, options)

cupy/cuda/compiler.py:511: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   cpdef compileProgram(intptr_t prog, options):

cupy_backends/cuda/libs/nvrtc.pyx:108: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   check_status(status)

cupy_backends/cuda/libs/nvrtc.pyx:120: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   raise NVRTCError(status)
E   cupy_backends.cuda.libs.nvrtc.NVRTCError: HIPRTC_ERROR_COMPILATION (6)

cupy_backends/cuda/libs/nvrtc.pyx:58: NVRTCError

During handling of the above exception, another exception occurred:

self = <cupy.testing._bundle.TestRaw_param_0_{backend='nvrtc', in_memory=False} testMethod=test_basic>, args = (), kwargs = {}
s = <_io.StringIO object at 0x7f9e2b7dcaf0>, k = 'in_memory', v = False, err_class = <class 'cupy.cuda.compiler.CompileException'>

    @functools.wraps(base_method)
    def new_method(self, *args, **kwargs):
        try:
>           return base_method(self, *args, **kwargs)

cupy/testing/parameterized.py:83: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <cupy.testing._bundle.TestRaw_param_0_{backend='nvrtc', in_memory=False} testMethod=test_basic>

    def test_basic(self):
>       x1, x2, y = self._helper(self.kern, cupy.float32)

tests/cupy_tests/core_tests/test_raw.py:438: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <cupy.testing._bundle.TestRaw_param_0_{backend='nvrtc', in_memory=False} testMethod=test_basic>, kernel = <cupy.core.raw.RawKernel object at 0x7f9e2c6ebc20>
dtype = <class 'numpy.float32'>

    def _helper(self, kernel, dtype):
        N = 10
>       x1 = cupy.arange(N**2, dtype=dtype).reshape(N, N)

tests/cupy_tests/core_tests/test_raw.py:431: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

start = 0, stop = 100, step = 1, dtype = <class 'numpy.float32'>

    def arange(start, stop=None, step=1, dtype=None):
        """Returns an array with evenly spaced values within a given interval.
    
        Values are generated within the half-open interval [start, stop). The first
        three arguments are mapped like the ``range`` built-in function, i.e. start
        and step are optional.
    
        Args:
            start: Start of the interval.
            stop: End of the interval.
            step: Step width between each pair of consecutive values.
            dtype: Data type specifier. It is inferred from other arguments by
                default.
    
        Returns:
            cupy.ndarray: The 1-D array of range values.
    
        .. seealso:: :func:`numpy.arange`
    
        """
        if dtype is None:
            if any(numpy.dtype(type(val)).kind == 'f'
                   for val in (start, stop, step)):
                dtype = float
            else:
                dtype = int
    
        if stop is None:
            stop = start
            start = 0
    
        if step is None:
            step = 1
    
        size = int(numpy.ceil((stop - start) / step))
        if size <= 0:
            return cupy.empty((0,), dtype=dtype)
    
        if numpy.dtype(dtype).type == numpy.bool_:
            if size > 2:
                raise ValueError('no fill-function for data-type.')
            if size == 2:
                return cupy.array([start, start - step], dtype=numpy.bool_)
            else:
                return cupy.array([start], dtype=numpy.bool_)
    
        ret = cupy.empty((size,), dtype=dtype)
        typ = numpy.dtype(dtype).type
>       _arange_ufunc(typ(start), typ(step), ret, dtype=dtype)

cupy/_creation/ranges.py:57: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   kern = self._get_ufunc_kernel(dev_id, op, arginfos)

cupy/core/_kernel.pyx:1084: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   kern = _get_ufunc_kernel(

cupy/core/_kernel.pyx:1107: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   return _get_simple_elementwise_kernel(

cupy/core/_kernel.pyx:881: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   module = compile_with_cache(module_code, options)

cupy/core/_kernel.pyx:60: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   return cuda.compile_with_cache(

cupy/core/core.pyx:1921: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '#include <cupy/complex.cuh>\n#include <cupy/carray.cuh>\n#include <cupy/atomics.cuh>\n\n    typedef float in0_type;\n...  _ind.set(i);\n        out0_type &out0 = _raw_out0[_ind.get()];\nout0 = in0 + i * in1;\n      }\n      ;\n    }\n    '
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include',), arch = None, cache_dir = None
extra_source = '#pragma once\n\n#include <cupy/complex/complex.h>\n\nusing thrust::complex;\nusing thrust::conj;\nusing thrust::real;...<float> atanh(const complex<float>& z) {\n  return detail::complex::catanhf(z);\n}\n#endif\n\n}  // namespace thrust\n'
backend = 'hiprtc'

    def compile_with_cache(
            source, options=(), arch=None, cache_dir=None, extra_source=None,
            backend='nvrtc', *, enable_cooperative_groups=False,
            name_expressions=None, log_stream=None):
    
        if enable_cooperative_groups:
            if backend != 'nvcc':
                raise ValueError(
                    'Cooperative groups is supported only in NVCC backend.')
            if runtime.is_hip:
                raise ValueError(
                    'Cooperative groups is not supported in HIP.')
    
        if name_expressions is not None:
            if runtime.is_hip or backend != 'nvrtc':
                raise NotImplementedError
    
        # We silently ignore CUPY_CACHE_IN_MEMORY if nvcc is in use, because it
        # must dump files to disk.
        # TODO(leofang): check if hiprtc can avoid disk access
        cache_in_memory = (
            _get_bool_env_variable('CUPY_CACHE_IN_MEMORY', False)
            and backend == 'nvrtc')
    
        if runtime.is_hip:
            backend = 'hiprtc' if backend == 'nvrtc' else 'hipcc'
            return _compile_with_cache_hip(
>               source, options, arch, cache_dir, extra_source, backend)

cupy/cuda/compiler.py:324: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '#include <hip/hip_runtime.h>\n#include <cupy/complex.cuh>\n#include <cupy/carray.cuh>\n#include <cupy/atomics.cuh>\n\...  _ind.set(i);\n        out0_type &out0 = _raw_out0[_ind.get()];\nout0 = in0 + i * in1;\n      }\n      ;\n    }\n    '
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include',), arch = 'gfx906', cache_dir = '/tmp/tmpmvpgm0e1'
extra_source = '#pragma once\n\n#include <cupy/complex/complex.h>\n\nusing thrust::complex;\nusing thrust::conj;\nusing thrust::real;...<float> atanh(const complex<float>& z) {\n  return detail::complex::catanhf(z);\n}\n#endif\n\n}  // namespace thrust\n'
backend = 'hiprtc', use_converter = True

    def _compile_with_cache_hip(source, options, arch, cache_dir, extra_source,
                                backend='hiprtc', use_converter=True):
        global _empty_file_preprocess_cache
        if cache_dir is None:
            cache_dir = get_cache_dir()
        if arch is None:
            arch = os.environ.get('HCC_AMDGPU_TARGET')
            if arch is None:
                raise RuntimeError('HCC_AMDGPU_TARGET is not set')
        if use_converter:
            source = _convert_to_hip_source(source)
    
        env = (arch, options, _get_hipcc_version())
        base = _empty_file_preprocess_cache.get(env, None)
        if base is None:
            # This is checking of HIPCC compiler internal version
            base = _preprocess_hipcc('', options)
            _empty_file_preprocess_cache[env] = base
        key_src = '%s %s %s %s' % (env, base, source, extra_source)
    
        key_src = key_src.encode('utf-8')
        name = '%s.hsaco' % hashlib.md5(key_src).hexdigest()
    
        if not os.path.isdir(cache_dir):
            try:
                os.makedirs(cache_dir)
            except OSError:
                if not os.path.isdir(cache_dir):
                    raise
    
        mod = function.Module()
        # To handle conflicts in concurrent situation, we adopt lock-free method
        # to avoid performance degradation.
        path = os.path.join(cache_dir, name)
        if os.path.exists(path):
            with open(path, 'rb') as file:
                data = file.read()
            if len(data) >= 32:
                hash_value = data[:32]
                binary = data[32:]
                binary_hash = hashlib.md5(binary).hexdigest().encode('ascii')
                if hash_value == binary_hash:
                    mod.load(binary)
                    return mod
    
        if backend == 'hiprtc':
            # compile_using_nvrtc calls hiprtc for hip builds
>           binary = compile_using_nvrtc(source, options, arch, name + '.cu')

cupy/cuda/compiler.py:662: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '#include <hip/hip_runtime.h>\n#include <cupy/complex.cuh>\n#include <cupy/carray.cuh>\n#include <cupy/atomics.cuh>\n\...  _ind.set(i);\n        out0_type &out0 = _raw_out0[_ind.get()];\nout0 = in0 + i * in1;\n      }\n      ;\n    }\n    '
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include', '-arch=compute_gfx906', '-I', '/home/leo/dev/cupy_rocm35/cupy/core/include'), arch = 'gfx906'
filename = 'f0164dd1669f20bf2c86da1c570546a3.hsaco.cu', name_expressions = None, log_stream = None, cache_in_memory = False

    def compile_using_nvrtc(source, options=(), arch=None, filename='kern.cu',
                            name_expressions=None, log_stream=None,
                            cache_in_memory=False):
        if not arch:
            arch = _get_arch()
    
        options += ('-arch=compute_{}'.format(arch),)
        options += ("-I", "/home/leo/dev/cupy_rocm35/cupy/core/include",)
        print(options)
    
        def _compile(source, options, cu_path, name_expressions, log_stream):
            prog = _NVRTCProgram(source, cu_path,
                                 name_expressions=name_expressions)
            try:
                ptx, mapping = prog.compile(options, log_stream)
            except CompileException as e:
                dump = _get_bool_env_variable(
                    'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
                if dump:
                    e.dump(sys.stderr)
                raise
            return ptx, mapping
    
        if not cache_in_memory:
            with tempfile.TemporaryDirectory() as root_dir:
                cu_path = os.path.join(root_dir, filename)
    
                with open(cu_path, 'w') as cu_file:
                    cu_file.write(source)
    
                return _compile(source, options, cu_path,
>                               name_expressions, log_stream)

cupy/cuda/compiler.py:164: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '#include <hip/hip_runtime.h>\n#include <cupy/complex.cuh>\n#include <cupy/carray.cuh>\n#include <cupy/atomics.cuh>\n\...  _ind.set(i);\n        out0_type &out0 = _raw_out0[_ind.get()];\nout0 = in0 + i * in1;\n      }\n      ;\n    }\n    '
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include', '-arch=compute_gfx906', '-I', '/home/leo/dev/cupy_rocm35/cupy/core/include')
cu_path = '/tmp/tmp45teprjn/f0164dd1669f20bf2c86da1c570546a3.hsaco.cu', name_expressions = None, log_stream = None

    def _compile(source, options, cu_path, name_expressions, log_stream):
        prog = _NVRTCProgram(source, cu_path,
                             name_expressions=name_expressions)
        try:
>           ptx, mapping = prog.compile(options, log_stream)

cupy/cuda/compiler.py:147: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <cupy.cuda.compiler._NVRTCProgram object at 0x7f9ef22db050>
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include', '-arch=compute_gfx906', '-I', '/home/leo/dev/cupy_rocm35/cupy/core/include'), log_stream = None

    def compile(self, options=(), log_stream=None):
        try:
            if self.name_expressions:
                for ker in self.name_expressions:
                    nvrtc.addAddNameExpression(self.ptr, ker)
            nvrtc.compileProgram(self.ptr, options)
            mapping = None
            if self.name_expressions:
                mapping = {}
                for ker in self.name_expressions:
                    mapping[ker] = nvrtc.getLoweredName(self.ptr, ker)
            if log_stream is not None:
                log_stream.write(nvrtc.getProgramLog(self.ptr))
            return nvrtc.getPTX(self.ptr), mapping
        except nvrtc.NVRTCError:
            log = nvrtc.getProgramLog(self.ptr)
>           raise CompileException(log, self.src, self.name, options, 'nvrtc')
E           cupy.cuda.compiler.CompileException: Error: Failed to compile opencl source (from CL or HIP source to LLVM IR).

cupy/cuda/compiler.py:522: CompileException

During handling of the above exception, another exception occurred:

self = <cupy.testing._bundle.TestRaw_param_0_{backend='nvrtc', in_memory=False} testMethod=test_basic>, args = (), kwargs = {}
s = <_io.StringIO object at 0x7f9e2b7dcaf0>, k = 'in_memory', v = False, err_class = <class 'cupy.cuda.compiler.CompileException'>

    @functools.wraps(base_method)
    def new_method(self, *args, **kwargs):
        try:
            return base_method(self, *args, **kwargs)
        except unittest.SkipTest:
            raise
        except Exception as e:
            s = io.StringIO()
            s.write('Parameterized test failed.\n\n')
            s.write('Base test method: {}.{}\n'.format(
                base.__name__, base_method.__name__))
            s.write('Test parameters:\n')
            for k, v in sorted(param.items()):
                s.write('  {}: {}\n'.format(k, v))
            err_class = e.__class__
            if isinstance(e, (driver.CUDADriverError, cufft.CuFFTError)):
                err_class, = err_class.__bases__
>           raise err_class(s.getvalue()).with_traceback(e.__traceback__)
E           TypeError: __init__() missing 3 required positional arguments: 'source', 'name', and 'options'

cupy/testing/parameterized.py:97: TypeError
------------------------------------------------------------------------ Captured stdout call -------------------------------------------------------------------------
('-I/home/leo/dev/cupy_rocm35/cupy/core/include', '-arch=compute_gfx906', '-I', '/home/leo/dev/cupy_rocm35/cupy/core/include')
amd_comgr_do_action:
	  ActionKind: AMD_COMGR_ACTION_COMPILE_SOURCE_TO_BC
	     IsaName: amdgcn-amd-amdhsa--gfx906+sram-ecc
	     Options: "-O3" "-D__HIP_ROCclr__=1" "-D__HIP_ARCH_GFX906__=1" "-mllvm" "-amdgpu-prelink" "-mcumode" "-mwavefrontsize64"
	        Path: 
	    Language: AMD_COMGR_LANGUAGE_HIP
COMGR::InProcessDriver::Execute argv: clang "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu" "-emit-llvm-bc" "-emit-llvm-uselists" "-disable-llvm-verifier" "-discard-value-names" "-main-file-name" "CompileSource" "-mrelocation-model" "pic" "-pic-level" "1" "-mthread-model" "posix" "-mframe-pointer=none" "-fdenormal-fp-math-f32=preserve-sign,preserve-sign" "-fno-rounding-math" "-mconstructor-aliases" "-aux-target-cpu" "x86-64" "-target-cpu" "gfx906" "-fcuda-is-device" "-fcuda-allow-variadic-functions" "-fvisibility" "hidden" "-fapply-global-visibility-to-externs" "-target-feature" "-wavefrontsize16" "-target-feature" "-wavefrontsize32" "-target-feature" "+wavefrontsize64" "-target-feature" "-xnack" "-target-feature" "+sram-ecc" "-target-feature" "+cumode" "-dwarf-column-info" "-fno-split-dwarf-inlining" "-debugger-tuning=gdb" "-resource-dir" "lib/clang/11.0.0" "-isystem" "/opt/rocm/hip/include" "-isystem" "/opt/rocm/llvm/lib/clang/11.0.0" "-isystem" "/opt/rocm/llvm/lib/clang/11.0.0/include" "-I" "/tmp/comgr-07e0aa/include" "-D" "__HIP_ROCclr__=1" "-D" "__HIP_ARCH_GFX906__=1" "-internal-isystem" "/usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0" "-internal-isystem" "/usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0" "-internal-isystem" "/usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0" "-internal-isystem" "/usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/backward" "-internal-isystem" "/usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0" "-internal-isystem" "/usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0" "-internal-isystem" "/usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0" "-internal-isystem" "/usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/backward" "-internal-isystem" "/usr/local/include" "-internal-isystem" "lib/clang/11.0.0/include" "-internal-externc-isystem" "/usr/include/x86_64-linux-gnu" "-internal-externc-isystem" "/include" "-internal-externc-isystem" "/usr/include" "-internal-isystem" "/usr/local/include" "-internal-isystem" "lib/clang/11.0.0/include" "-internal-externc-isystem" "/usr/include/x86_64-linux-gnu" "-internal-externc-isystem" "/include" "-internal-externc-isystem" "/usr/include" "-O3" "-std=c++11" "-fdeprecated-macro" "-fno-autolink" "-fdebug-compilation-dir" "/home/leo/dev/cupy_rocm35" "-ferror-limit" "19" "-fgnuc-version=4.2.1" "-fcxx-exceptions" "-fexceptions" "-vectorize-loops" "-vectorize-slp" "-mllvm" "-amdgpu-prelink" "-fcuda-allow-variadic-functions" "-faddrsig" "-o" "/tmp/comgr-07e0aa/output/CompileSource.bc" "-x" "hip" "/tmp/comgr-07e0aa/input/CompileSource"
/tmp/comgr-07e0aa/input/CompileSource:2:10: fatal error: 'cupy/complex.cuh' file not found
#include <cupy/complex.cuh>
         ^~~~~~~~~~~~~~~~~~
------------------------------------------------------------------------ Captured stderr call -------------------------------------------------------------------------
NVRTC compilation error: Error: Failed to compile opencl source (from CL or HIP source to LLVM IR).

-----
Name: /tmp/tmp45teprjn/f0164dd1669f20bf2c86da1c570546a3.hsaco.cu
Options: -I/home/leo/dev/cupy_rocm35/cupy/core/include -arch=compute_gfx906 -I /home/leo/dev/cupy_rocm35/cupy/core/include
CUDA source:
01 #include <hip/hip_runtime.h>
02 #include <cupy/complex.cuh>
03 #include <cupy/carray.cuh>
04 #include <cupy/atomics.cuh>
05 
06     typedef float in0_type;
07 typedef float in1_type;
08 typedef float out0_type;
09 
10 
11     extern "C" __global__ void cupy_arange__float_float_float32(const float in0, const float in1, CArray<float, 1, 1, 1> _raw_out0, CIndexer<1> _ind) {
12       ;
13       CUPY_FOR(i, _ind.size()) {
14         _ind.set(i);
15         out0_type &out0 = _raw_out0[_ind.get()];
16 out0 = in0 + i * in1;
17       }
18       ;
19     }
20 
-----
======================================================================= short test summary info =======================================================================
FAILED tests/cupy_tests/core_tests/test_raw.py::TestRaw_param_0_{backend='nvrtc', in_memory=False}::test_basic - TypeError: __init__() missing 3 required positional...
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
========================================================================== 1 failed in 1.56s ==========================================================================
1 error generated when compiling for gfx906.
	ReturnStatus: AMD_COMGR_STATUS_ERROR
``` 


Discussion:

Resolving patch:
diff --git a/cupy/cuda/compiler.py b/cupy/cuda/compiler.py
--- a/cupy/cuda/compiler.py
+++ b/cupy/cuda/compiler.py
@@ -25,28 +25,40 @@ class NVCCException(Exception):
     pass
 
 
-def _run_nvcc(cmd, cwd, log_stream=None):
+class HIPCCException(Exception):
+    pass
+
+
+def _run_cc(cmd, cwd, backend, log_stream=None):
+    # backend in ('nvcc', 'hipcc')
     try:
-        log = subprocess.check_output(cmd, cwd=cwd,
+        env = os.environ if backend == 'hipcc' else None
+        log = subprocess.check_output(cmd, cwd=cwd, env=env,
                                       stderr=subprocess.STDOUT,
                                       universal_newlines=True)
         if log_stream is not None:
             log_stream.write(log)
         return log
     except subprocess.CalledProcessError as e:
-        msg = ('`nvcc` command returns non-zero exit status. \n'
-               'command: {0}\n'
-               'return-code: {1}\n'
+        msg = ('`{0}` command returns non-zero exit status. \n'
+               'command: {1}\n'
+               'return-code: {2}\n'
                'stdout/stderr: \n'
-               '{2}'.format(e.cmd,
+               '{3}'.format(backend,
+                            e.cmd,
                             e.returncode,
                             e.output))
-        raise NVCCException(msg)
+        if backend == 'nvcc':
+            raise NVCCException(msg)
+        elif backend == 'hipcc':
+            raise HIPCCException(msg)
+        else:
+            raise RuntimeError(msg)
     except OSError as e:
-        msg = 'Failed to run `nvcc` command. ' \
+        msg = 'Failed to run `{0}` command. ' \
               'Check PATH environment variable: ' \
               + str(e)
-        raise OSError(msg)
+        raise OSError(msg.format(backend))
 
 
 def _get_nvrtc_version():
@@ -200,7 +212,7 @@ def compile_using_nvcc(source, options=(), arch=None,
             cmd.append(cu_path)
 
             try:
-                _run_nvcc(cmd, root_dir, log_stream)
+                _run_cc(cmd, root_dir, 'nvcc', log_stream)
             except NVCCException as e:
                 cex = CompileException(str(e), source, cu_path, options,
                                        'nvcc')
@@ -220,7 +232,7 @@ def compile_using_nvcc(source, options=(), arch=None,
             cmd.append(cu_path)
 
             try:
-                _run_nvcc(cmd, root_dir, log_stream)
+                _run_cc(cmd, root_dir, 'nvcc', log_stream)
             except NVCCException as e:
                 cex = CompileException(str(e), source, cu_path, options,
                                        'nvcc')
@@ -237,7 +249,7 @@ def compile_using_nvcc(source, options=(), arch=None,
             cmd = cmd_partial + list(options)
 
             try:
-                _run_nvcc(cmd, root_dir, log_stream)
+                _run_cc(cmd, root_dir, 'nvcc', log_stream)
             except NVCCException as e:
                 cex = CompileException(str(e), '', '', options, 'nvcc')
                 raise cex
@@ -305,13 +317,11 @@ def compile_with_cache(
             raise ValueError(
                 'Cooperative groups is not supported in HIP.')
 
-    if name_expressions is not None:
-        if runtime.is_hip or backend != 'nvrtc':
-            raise NotImplementedError
+    if name_expressions is not None and backend != 'nvrtc':
+        raise NotImplementedError
 
-    # We silently ignore CUPY_CACHE_IN_MEMORY if nvcc is in use, because it
-    # must dump files to disk.
-    # TODO(leofang): check if hiprtc can avoid disk access
+    # We silently ignore CUPY_CACHE_IN_MEMORY if nvcc/hipcc are in use, because
+    # they must dump files to disk.
     cache_in_memory = (
         _get_bool_env_variable('CUPY_CACHE_IN_MEMORY', False)
         and backend == 'nvrtc')
@@ -319,7 +329,8 @@ def compile_with_cache(
     if runtime.is_hip:
         backend = 'hiprtc' if backend == 'nvrtc' else 'hipcc'
         return _compile_with_cache_hip(
-            source, options, arch, cache_dir, extra_source, backend)
+            source, options, arch, cache_dir, extra_source, backend,
+            name_expressions, log_stream, cache_in_memory)
     else:
         return _compile_with_cache_cuda(
             source, options, arch, cache_dir, extra_source, backend,
@@ -364,13 +375,8 @@ def _compile_with_cache_cuda(
 
     if not cache_in_memory:
         # Read from disk cache
-
         if not os.path.isdir(cache_dir):
-            try:
-                os.makedirs(cache_dir)
-            except OSError:
-                if not os.path.isdir(cache_dir):
-                    raise
+            os.makedirs(cache_dir, exist_ok=True)
 
         # To handle conflicts in concurrent situation, we adopt lock-free
         # method to avoid performance degradation.
@@ -415,7 +421,6 @@ def _compile_with_cache_cuda(
 
     if not cache_in_memory:
         # Write to disk cache
-
         cubin_hash = hashlib.md5(cubin).hexdigest().encode('ascii')
 
         # shutil.move is not atomic operation, so it could result in a
@@ -517,7 +522,8 @@ def compile(self, options=(), log_stream=None):
             return nvrtc.getPTX(self.ptr), mapping
         except nvrtc.NVRTCError:
             log = nvrtc.getProgramLog(self.ptr)
-            raise CompileException(log, self.src, self.name, options, 'nvrtc')
+            raise CompileException(log, self.src, self.name, options,
+                                   'nvrtc' if not runtime.is_hip else 'hiprtc')
 
 
 def is_valid_kernel_name(name):
@@ -531,29 +537,11 @@ def _get_hipcc_version():
     global _hipcc_version
     if _hipcc_version is None:
         cmd = ['hipcc', '--version']
-        _hipcc_version = _run_hipcc(cmd)
+        _hipcc_version = _run_cc(cmd, '.', 'hipcc')
     return _hipcc_version
 
 
-def _run_hipcc(cmd, cwd='.', env=None):
-    try:
-        return subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd,
-                                       env=env)
-    except subprocess.CalledProcessError as e:
-        # TODO(leofang): raise an "HIPCCException"?
-        raise RuntimeError(
-            '`hipcc` command returns non-zero exit status. \n'
-            'command: {0}\n'
-            'return-code: {1}\n'
-            'stdout/stderr: \n'
-            '{2}'.format(e.cmd, e.returncode, e.output.decode('utf-8')))
-    except OSError as e:
-        raise OSError('Failed to run `hipcc` command. '
-                      'Check PATH environment variable: '
-                      + str(e))
-
-
-def _hipcc(source, options, arch):
+def compile_using_hipcc(source, options, arch, log_stream=None):
     assert len(arch) > 0
     # pass HCC_AMDGPU_TARGET same as arch
     cmd = ['hipcc', '--genco'] + list(options)
@@ -568,11 +556,20 @@ def _hipcc(source, options, arch):
 
         cmd += [in_path, '-o', out_path]
 
-        env = os.environ.copy()
+        try:
+            output = _run_cc(cmd, root_dir, 'hipcc', log_stream)
+        except HIPCCException as e:
+            cex = CompileException(str(e), source, in_path, options,
+                                   'hipcc')
 
-        output = _run_hipcc(cmd, root_dir, env)
+            dump = _get_bool_env_variable(
+                'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
+            if dump:
+                cex.dump(sys.stderr)
+
+            raise cex
         if not os.path.isfile(out_path):
-            raise RuntimeError(
+            raise HIPCCException(
                 '`hipcc` command does not generate output file. \n'
                 'command: {0}\n'
                 'stdout/stderr: \n'
@@ -591,12 +588,15 @@ def _preprocess_hipcc(source, options):
             cu_file.write(source)
 
         cmd.append(cu_path)
-        pp_src = _run_hipcc(cmd, root_dir)
-        assert isinstance(pp_src, bytes)
-        return re.sub(b'(?m)^#.*$', b'', pp_src)
+        pp_src = _run_cc(cmd, root_dir, 'hipcc')
+        assert isinstance(pp_src, str)
+        return re.sub('(?m)^#.*$', '', pp_src)
+
+
+_hip_extra_source = None
 
 
-def _convert_to_hip_source(source):
+def _convert_to_hip_source(source, extra_source, is_hiprtc):
     table = [
         ('threadIdx.', 'hipThreadIdx_'),
         ('blockIdx.', 'hipBlockIdx_'),
@@ -605,21 +605,51 @@ def _convert_to_hip_source(source):
     ]
     for i, j in table:
         source = source.replace(i, j)
+    if not is_hiprtc:
+        return '#include <hip/hip_runtime.h>\n' + source
 
-    return "#include <hip/hip_runtime.h>\n" + source
+    # Workaround for hiprtc: it does not follow the -I option to search
+    # headers (as of ROCm 3.5.0), so we must prepend all CuPy's headers
+    global _hip_extra_source
+    if _hip_extra_source is None:
+        if extra_source is not None:
+            extra_source = extra_source.split('\n')
+            extra_source = [line for line in extra_source if (
+                not line.startswith('#include')
+                and not line.startswith('#pragma once'))]
+            _hip_extra_source = extra_source = '\n'.join(extra_source)
 
+    source = source.split('\n')
+    source = [line for line in source if not line.startswith('#include')]
+    source = ('#include <hip/hip_runtime.h>\n#include <hip/hip_fp16.h>\n'
+              + _hip_extra_source + '\n'.join(source))
 
+    return source
+
+
+# TODO(leofang): evaluate if this can be merged with _compile_with_cache_cuda()
 def _compile_with_cache_hip(source, options, arch, cache_dir, extra_source,
-                            backend='hiprtc', use_converter=True):
+                            backend='hiprtc', name_expressions=None,
+                            log_stream=None, cache_in_memory=False,
+                            use_converter=True):
     global _empty_file_preprocess_cache
+
+    # TODO(leofang): this might be possible but is currently undocumented
+    if _is_cudadevrt_needed(options):
+        raise ValueError('separate compilation is not supported in HIP')
+
     if cache_dir is None:
         cache_dir = get_cache_dir()
+    # TODO(leofang): it seems as of ROCm 3.5.0 hiprtc/hipcc can automatically
+    # pick up the right arch without needing HCC_AMDGPU_TARGET. Check the
+    # earliest ROCm version in which this happened.
     if arch is None:
         arch = os.environ.get('HCC_AMDGPU_TARGET')
         if arch is None:
             raise RuntimeError('HCC_AMDGPU_TARGET is not set')
     if use_converter:
-        source = _convert_to_hip_source(source)
+        source = _convert_to_hip_source(source, extra_source,
+                                        is_hiprtc=(backend == 'hiprtc'))
 
     env = (arch, options, _get_hipcc_version())
     base = _empty_file_preprocess_cache.get(env, None)
@@ -627,55 +657,67 @@ def _compile_with_cache_hip(source, options, arch, cache_dir, extra_source,
         # This is checking of HIPCC compiler internal version
         base = _preprocess_hipcc('', options)
         _empty_file_preprocess_cache[env] = base
-    key_src = '%s %s %s %s' % (env, base, source, extra_source)
 
+    key_src = '%s %s %s %s' % (env, base, source, extra_source)
     key_src = key_src.encode('utf-8')
     name = '%s.hsaco' % hashlib.md5(key_src).hexdigest()
 
-    if not os.path.isdir(cache_dir):
-        try:
-            os.makedirs(cache_dir)
-        except OSError:
-            if not os.path.isdir(cache_dir):
-                raise
-
     mod = function.Module()
-    # To handle conflicts in concurrent situation, we adopt lock-free method
-    # to avoid performance degradation.
-    path = os.path.join(cache_dir, name)
-    if os.path.exists(path):
-        with open(path, 'rb') as file:
-            data = file.read()
-        if len(data) >= 32:
-            hash_value = data[:32]
-            binary = data[32:]
-            binary_hash = hashlib.md5(binary).hexdigest().encode('ascii')
-            if hash_value == binary_hash:
-                mod.load(binary)
-                return mod
+
+    if not cache_in_memory:
+        # Read from disk cache
+        if not os.path.isdir(cache_dir):
+            os.makedirs(cache_dir, exist_ok=True)
+
+        # To handle conflicts in concurrent situation, we adopt lock-free
+        # method to avoid performance degradation.
+        # We force recompiling to retrieve C++ mangled names if so desired.
+        path = os.path.join(cache_dir, name)
+        if os.path.exists(path) and not name_expressions:
+            with open(path, 'rb') as f:
+                data = f.read()
+            if len(data) >= 32:
+                hash_value = data[:32]
+                binary = data[32:]
+                binary_hash = hashlib.md5(binary).hexdigest().encode('ascii')
+                if hash_value == binary_hash:
+                    mod.load(binary)
+                    return mod
+    else:
+        # Enforce compiling -- the resulting kernel will be cached elsewhere,
+        # so we do nothing
+        pass
 
     if backend == 'hiprtc':
         # compile_using_nvrtc calls hiprtc for hip builds
-        binary = compile_using_nvrtc(source, options, arch, name + '.cu')
+        binary, mapping = compile_using_nvrtc(
+            source, options, arch, name + '.cu', name_expressions,
+            log_stream, cache_in_memory)
+        mod._set_mapping(mapping)
     else:
-        # TODO(leofang): catch HIPCCException and convert it to
-        # CompileException with backend='hipcc'
-        binary = _hipcc(source, options, arch)
-    binary_hash = hashlib.md5(binary).hexdigest().encode('ascii')
-
-    # shutil.move is not atomic operation, so it could result in a corrupted
-    # file. We detect it by appending md5 hash at the beginning of each cache
-    # file. If the file is corrupted, it will be ignored next time it is read.
-    with tempfile.NamedTemporaryFile(dir=cache_dir, delete=False) as tf:
-        tf.write(binary_hash)
-        tf.write(binary)
-        temp_path = tf.name
-    shutil.move(temp_path, path)
-
-    # Save .cu source file along with .hsaco
-    if _get_bool_env_variable('CUPY_CACHE_SAVE_CUDA_SOURCE', False):
-        with open(path + '.cu', 'w') as f:
-            f.write(source)
+        binary = compile_using_hipcc(source, options, arch, log_stream)
+
+    if not cache_in_memory:
+        # Write to disk cache
+        binary_hash = hashlib.md5(binary).hexdigest().encode('ascii')
+
+        # shutil.move is not atomic operation, so it could result in a
+        # corrupted file. We detect it by appending md5 hash at the beginning
+        # of each cache file. If the file is corrupted, it will be ignored
+        # next time it is read.
+        with tempfile.NamedTemporaryFile(dir=cache_dir, delete=False) as tf:
+            tf.write(binary_hash)
+            tf.write(binary)
+            temp_path = tf.name
+        shutil.move(temp_path, path)
+
+        # Save .cu source file along with .hsaco
+        if _get_bool_env_variable('CUPY_CACHE_SAVE_CUDA_SOURCE', False):
+            with open(path + '.cpp', 'w') as f:
+                f.write(source)
+    else:
+        # we don't do any disk I/O
+        pass
 
     mod.load(binary)
     return mod


Test patch:
diff --git a/tests/cupy_tests/core_tests/test_cub_reduction.py b/tests/cupy_tests/core_tests/test_cub_reduction.py
--- a/tests/cupy_tests/core_tests/test_cub_reduction.py
+++ b/tests/cupy_tests/core_tests/test_cub_reduction.py
@@ -124,6 +124,7 @@ def test_can_use_cub_oversize_input4(self):
         b = cupy.empty((), dtype=cupy.int8)
         assert self.can_use([a], [b], (1,), (0,)) is None
 
+    @unittest.skipIf(cupy.cuda.runtime.is_hip, 'hip does not support CUB')
     def test_can_use_accelerator_set_unset(self):
         # ensure we use CUB block reduction and not CUB device reduction
         old_routine_accelerators = _accelerator.get_routine_accelerators()
diff --git a/tests/cupy_tests/core_tests/test_raw.py b/tests/cupy_tests/core_tests/test_raw.py
--- a/tests/cupy_tests/core_tests/test_raw.py
+++ b/tests/cupy_tests/core_tests/test_raw.py
@@ -451,7 +451,9 @@ def test_kernel_attributes(self):
                           'ptx_version',
                           'shared_size_bytes']:
             assert attribute in attrs
-        assert self.kern.num_regs > 0
+        # TODO(leofang): investigate why this fails on ROCm 3.5.0
+        if not cupy.cuda.runtime.is_hip:
+            assert self.kern.num_regs > 0
         assert self.kern.max_threads_per_block > 0
         assert self.kern.shared_size_bytes == 0
 
@@ -478,6 +480,9 @@ def test_compiler_flag(self):
         assert cupy.allclose(y, x1 * x2)
 
     def test_invalid_compiler_flag(self):
+        if cupy.cuda.runtime.is_hip and self.backend == 'nvrtc':
+            self.skipTest('hiprtc does not handle #error macro properly')
+
         with pytest.raises(cupy.cuda.compiler.CompileException) as ex:
             mod = cupy.RawModule(code=_test_source3,
                                  options=('-DPRECISION=3',),
@@ -486,30 +491,44 @@ def test_invalid_compiler_flag(self):
         assert 'precision not supported' in str(ex.value)
 
     def _generate_file(self, ext: str):
-        # generate cubin/ptx by calling nvcc
+        # generate cubin/ptx by calling nvcc/hipcc
 
-        nvcc = cupy.cuda.get_nvcc_path()
+        if not cupy.cuda.runtime.is_hip:
+            cc = cupy.cuda.get_nvcc_path()
+            arch = '-gencode=arch=compute_{CC},code=sm_{CC}'.format(
+                CC=compiler._get_arch())
+            code = _test_source5
+        else:
+            # TODO(leofang): We currently instruct users to set up
+            # $ROCM_HOME, but perhaps it could be relaxed after we add
+            # get_hipcc_path().
+            cc = os.path.join(os.environ['ROCM_HOME'], 'bin/hipcc')
+            arch = '-v'  # dummy
+            code = compiler._convert_to_hip_source(_test_source5, None, False)
         # split() is needed because nvcc could come from the env var NVCC
-        cmd = nvcc.split()
-        arch = '-gencode=arch=compute_{cc},code=sm_{cc}'.format(
-            cc=compiler._get_arch())
+        cmd = cc.split()
         source = '{}/test_load_cubin.cu'.format(self.cache_dir)
         file_path = self.cache_dir + 'test_load_cubin'
         with open(source, 'w') as f:
-            f.write(_test_source5)
+            f.write(code)
         if ext == 'cubin':
             file_path += '.cubin'
             flag = '-cubin'
         elif ext == 'ptx':
             file_path += '.ptx'
             flag = '-ptx'
+        elif ext == 'hsaco':
+            file_path += '.hsaco'
+            flag = '--genco'
         else:
             raise ValueError
         cmd += [arch, flag, source, '-o', file_path]
-        compiler._run_nvcc(cmd, self.cache_dir)
+        cc = 'nvcc' if not cupy.cuda.runtime.is_hip else 'hipcc'
+        compiler._run_cc(cmd, self.cache_dir, cc)
 
         return file_path
 
+    @unittest.skipIf(cupy.cuda.runtime.is_hip, 'HIP uses hsaco, not cubin')
     def test_load_cubin(self):
         # generate cubin in the temp dir
         file_path = self._generate_file('cubin')
@@ -520,6 +539,7 @@ def test_load_cubin(self):
         x1, x2, y = self._helper(ker, cupy.float32)
         assert cupy.allclose(y, x1 / (x2 + 1.0))
 
+    @unittest.skipIf(cupy.cuda.runtime.is_hip, 'HIP uses hsaco, not ptx')
     def test_load_ptx(self):
         # generate ptx in the temp dir
         file_path = self._generate_file('ptx')
@@ -530,6 +550,18 @@ def test_load_ptx(self):
         x1, x2, y = self._helper(ker, cupy.float32)
         assert cupy.allclose(y, x1 / (x2 + 1.0))
 
+    @unittest.skipIf(not cupy.cuda.runtime.is_hip,
+                     'CUDA uses cubin/ptx, not hsaco')
+    def test_load_hsaco(self):
+        # generate hsaco in the temp dir
+        file_path = self._generate_file('hsaco')
+
+        # load cubin and test the kernel
+        mod = cupy.RawModule(path=file_path, backend=self.backend)
+        ker = mod.get_function('test_div')
+        x1, x2, y = self._helper(ker, cupy.float32)
+        assert cupy.allclose(y, x1 / (x2 + 1.0))
+
     def test_module_load_failure(self):
         # in principle this test is better done in test_driver.py, but
         # this error is more likely to appear when using RawModule, so
@@ -539,7 +571,8 @@ def test_module_load_failure(self):
                 path=os.path.expanduser('~/this_does_not_exist.cubin'),
                 backend=self.backend)
             mod.get_function('nonexisting_kernel')  # enforce loading
-        assert 'CUDA_ERROR_FILE_NOT_FOUND' in str(ex.value)
+        assert ('CUDA_ERROR_FILE_NOT_FOUND' in str(ex.value)  # CUDA
+                or 'hipErrorFileNotFound' in str(ex.value))  # HIP
 
     def test_module_neither_code_nor_path(self):
         with pytest.raises(TypeError):
@@ -557,8 +590,11 @@ def test_get_function_failure(self):
         # let us do it here
         with pytest.raises(cupy.cuda.driver.CUDADriverError) as ex:
             self.mod2.get_function('no_such_kernel')
-        assert 'CUDA_ERROR_NOT_FOUND' in str(ex.value)
+        assert ('CUDA_ERROR_NOT_FOUND' in str(ex.value)  # for CUDA
+                or 'hipErrorNotFound' in str(ex.value))  # for HIP
 
+    @unittest.skipIf(cupy.cuda.runtime.is_hip,
+                     'ROCm/HIP does not support dynamic parallelism')
     def test_dynamical_parallelism(self):
         ker = cupy.RawKernel(_test_source4, 'test_kernel', options=('-dc',),
                              backend=self.backend)
@@ -575,14 +611,16 @@ def test_dynamical_parallelism_compile_failure(self):
         N = 10
         inner_chunk = 2
         x = cupy.zeros((N,), dtype=cupy.float32)
-        if self.backend == 'nvrtc':
+        if self.backend == 'nvrtc' and not cupy.cuda.runtime.is_hip:
             # raised when calling ls.complete()
-            with pytest.raises(cupy.cuda.driver.CUDADriverError):
-                ker((1,), (N//inner_chunk,), (x, N, inner_chunk))
-        else:  # nvcc
-            with pytest.raises(cupy.cuda.compiler.CompileException):
-                ker((1,), (N//inner_chunk,), (x, N, inner_chunk))
-
+            error = cupy.cuda.driver.CUDADriverError
+        else:  # nvcc, hipcc, hiprtc
+            error = cupy.cuda.compiler.CompileException
+        with pytest.raises(error):
+            ker((1,), (N//inner_chunk,), (x, N, inner_chunk))
+
+    @unittest.skipIf(cupy.cuda.runtime.is_hip,
+                     'HIP code should not use cuFloatComplex')
     def test_cuFloatComplex(self):
         N = 100
         block = 32
@@ -645,6 +683,8 @@ def test_cuFloatComplex(self):
         ker((grid,), (block,), (a, b, out))
         assert (out == a + b).all()
 
+    @unittest.skipIf(cupy.cuda.runtime.is_hip,
+                     'HIP code should not use cuDoubleComplex')
     def test_cuDoubleComplex(self):
         N = 100
         block = 32
@@ -727,6 +767,12 @@ def test_template_specialization(self):
         if self.backend == 'nvcc':
             self.skipTest('nvcc does not support template specialization')
 
+        # TODO(leofang): investigate why hiprtc generates a wrong source code
+        # when the same code is compiled and discarded. It seems hiprtc has
+        # an internal cache that conflicts with the 2nd compilation attempt.
+        if cupy.cuda.runtime.is_hip and hasattr(self, 'clean_up'):
+            self.skipTest('skip a potential hiprtc bug')
+
         # compile code
         name_expressions = ['my_sqrt<int>', 'my_sqrt<float>',
                             'my_sqrt<complex<double>>', 'my_func']
@@ -736,6 +782,12 @@ def test_template_specialization(self):
         dtypes = (cupy.int32, cupy.float32, cupy.complex128, cupy.float64)
         for ker_T, dtype in zip(name_expressions, dtypes):
             # get specialized kernels
+            if cupy.cuda.runtime.is_hip:
+                # TODO(leofang): investigate why getLoweredName has no error
+                # but returns an empty string for my_sqrt<complex<double>>
+                mangled_name = mod.module.mapping.get(ker_T)
+                if mangled_name == '':
+                    continue
             ker = mod.get_function(ker_T)
 
             # prepare inputs & expected outputs
@@ -763,8 +815,9 @@ def test_template_failure(self):
         # 2. compile code without specializations
         mod = cupy.RawModule(code=test_cxx_template, options=('--std=c++11',))
         # ...try to get a specialized kernel
-        with pytest.raises(cupy.cuda.driver.CUDADriverError,
-                           match='named symbol not found'):
+        match = ('named symbol not found' if not cupy.cuda.runtime.is_hip else
+                 'hipErrorNotFound')
+        with pytest.raises(cupy.cuda.driver.CUDADriverError, match=match):
             mod.get_function('my_sqrt<int>')
 
         # 3. compile code without specifying C++ standard
@@ -775,8 +828,11 @@ def test_template_failure(self):
         # 4. try to fetch something we didn't specialize for
         mod = cupy.RawModule(code=test_cxx_template, options=('--std=c++11',),
                              name_expressions=name_expressions)
-        with pytest.raises(cupy.cuda.driver.CUDADriverError,
-                           match='named symbol not found'):
+        if cupy.cuda.runtime.is_hip:
+            msg = 'hipErrorNotFound'
+        else:
+            msg = 'named symbol not found'
+        with pytest.raises(cupy.cuda.driver.CUDADriverError, match=msg):
             mod.get_function('my_sqrt<double>')
 
     def test_raw_pointer(self):
@@ -932,22 +988,13 @@ def test_context_switch_RawModule6(self):
             # check results
             assert cupy.allclose(in_arr, out_arr)
 
-
-class TestCompile(unittest.TestCase):
-
-    def _helper(self, kernel, dtype):
-        N = 10
-        x1 = cupy.arange(N**2, dtype=dtype).reshape(N, N)
-        x2 = cupy.ones((N, N), dtype=dtype)
-        y = cupy.zeros((N, N), dtype=dtype)
-        kernel((N,), (N,), (x1, x2, y, N**2))
-        return x1, x2, y
-
+    @unittest.skipUnless(not cupy.cuda.runtime.is_hip,
+                         'only CUDA raises warning')
     def test_compile_kernel(self):
         kern = cupy.RawKernel(
             _test_compile_src, 'test_op',
             options=('-DOP=+',),
-            backend='nvcc')
+            backend=self.backend)
         log = io.StringIO()
         with use_temporary_cache_dir():
             kern.compile(log_stream=log)
@@ -955,10 +1002,12 @@ def test_compile_kernel(self):
         x1, x2, y = self._helper(kern, cupy.float32)
         assert cupy.allclose(y, x1 + x2)
 
+    @unittest.skipUnless(not cupy.cuda.runtime.is_hip,
+                         'only CUDA raises warning')
     def test_compile_module(self):
         module = cupy.RawModule(
             code=_test_compile_src,
-            backend='nvcc',
+            backend=self.backend,
             options=('-DOP=+',))
         log = io.StringIO()
         with use_temporary_cache_dir():
diff --git a/tests/cupy_tests/test_cusparse.py b/tests/cupy_tests/test_cusparse.py
--- a/tests/cupy_tests/test_cusparse.py
+++ b/tests/cupy_tests/test_cusparse.py
@@ -426,6 +426,9 @@ def test_csrmvEx_with_y(self):
 class TestCoosort(unittest.TestCase):
 
     def setUp(self):
+        if not cusparse.check_availability('coosort'):
+            pytest.skip('coosort is not available')
+
         self.a = scipy.sparse.random(
             100, 100, density=0.9, dtype=numpy.float32, format='coo')
         numpy.random.shuffle(self.a.row)
@@ -454,6 +457,9 @@ def test_coosort_by_column(self):
 class TestCsrsort(unittest.TestCase):
 
     def setUp(self):
+        if not cusparse.check_availability('csrsort'):
+            pytest.skip('csrsort is not available')
+
         self.a = scipy.sparse.random(
             1, 1000, density=0.9, dtype=numpy.float32, format='csr')
         numpy.random.shuffle(self.a.indices)
@@ -473,6 +479,9 @@ def test_csrsort(self):
 class TestCscsort(unittest.TestCase):
 
     def setUp(self):
+        if not cusparse.check_availability('cscsort'):
+            pytest.skip('cscsort is not available')
+
         self.a = scipy.sparse.random(
             1000, 1, density=0.9, dtype=numpy.float32, format='csc')
         numpy.random.shuffle(self.a.indices)