Repository: cupy/cupy

Issue:
ROCm 3.5.0 test: test_searchsorted
Got a linker error for many `searchsorted` tests...Not sure if this is an upstream (LLVM) bug or not...
```
$ pytest tests/cupy_tests/sorting_tests/test_search.py -v -rA -x -k test_searchsorted
========================================================================= test session starts =========================================================================
platform linux -- Python 3.7.8, pytest-6.0.1, py-1.9.0, pluggy-0.13.1 -- /home/leo/miniconda3/envs/cupy_rocm35_dev/bin/python
cachedir: .pytest_cache
rootdir: /home/leo/dev/cupy_rocm35, configfile: setup.cfg
collected 189 items / 108 deselected / 81 selected                                                                                                                    

tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_0_{bins=[], shape=(), side='left'}::test_searchsorted PASSED                              [  1%]
tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_1_{bins=[], shape=(), side='right'}::test_searchsorted PASSED                             [  2%]
tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_2_{bins=[], shape=(10,), side='left'}::test_searchsorted PASSED                           [  3%]
tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_3_{bins=[], shape=(10,), side='right'}::test_searchsorted PASSED                          [  4%]
tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_4_{bins=[], shape=(6, 3, 3), side='left'}::test_searchsorted PASSED                       [  6%]
tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_5_{bins=[], shape=(6, 3, 3), side='right'}::test_searchsorted PASSED                      [  7%]
tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_6_{bins=[0, 1, 2, 4, 10], shape=(), side='left'}::test_searchsorted PASSED                [  8%]
tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_7_{bins=[0, 1, 2, 4, 10], shape=(), side='right'}::test_searchsorted PASSED               [  9%]
tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'}::test_searchsorted FAILED             [ 11%]

============================================================================== FAILURES ===============================================================================
____________________________________ TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'}.test_searchsorted _____________________________________

self = <cupy.testing._bundle.TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'} testMethod=test_searchsorted>, args = (), kwargs = {}
s = <_io.StringIO object at 0x7f6ecc3379b0>, k = 'side', v = 'left', err_class = <class 'AssertionError'>

    @functools.wraps(base_method)
    def new_method(self, *args, **kwargs):
        try:
>           return base_method(self, *args, **kwargs)

cupy/testing/parameterized.py:83: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <cupy.testing._bundle.TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'} testMethod=test_searchsorted>, args = ()
kw = {'dtype': <class 'numpy.complex64'>}, dtype = <class 'numpy.complex64'>

    @functools.wraps(impl)
    def test_func(self, *args, **kw):
        for dtype in dtypes:
            try:
                kw[name] = numpy.dtype(dtype).type
>               impl(self, *args, **kw)

cupy/testing/helper.py:704: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <cupy.testing._bundle.TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'} testMethod=test_searchsorted>, args = ()
kw = {'dtype': <class 'numpy.complex64'>}, cupy_result = None
cupy_error = `hipcc` command returns non-zero exit status. 
command: ['hipcc', '--genco', '-I/opt/rocm/include', '-I/home/leo/dev/c...n-f4d106.cu
clang-11: note: diagnostic msg: /tmp/kern-f4d106.sh
clang-11: note: diagnostic msg: 

********************

cupy_tb = <traceback object at 0x7f6ecc1ea5f0>, numpy_result = (array([2, 3, 3, 4, 4, 4, 4, 4, 4, 5]),), numpy_error = None, numpy_tb = None

    @functools.wraps(impl)
    def test_func(self, *args, **kw):
        # Run cupy and numpy
        (
            cupy_result, cupy_error, cupy_tb,
            numpy_result, numpy_error, numpy_tb) = (
                _call_func_numpy_cupy(
                    self, impl, args, kw, name, sp_name, scipy_name))
        assert cupy_result is not None or cupy_error is not None
        assert numpy_result is not None or numpy_error is not None
    
        # Check errors raised
        if cupy_error or numpy_error:
            _check_cupy_numpy_error(self, cupy_error, cupy_tb,
                                    numpy_error, numpy_tb,
>                                   accept_error=accept_error)

cupy/testing/helper.py:217: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <cupy.testing._bundle.TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'} testMethod=test_searchsorted>
cupy_error = `hipcc` command returns non-zero exit status. 
command: ['hipcc', '--genco', '-I/opt/rocm/include', '-I/home/leo/dev/c...n-f4d106.cu
clang-11: note: diagnostic msg: /tmp/kern-f4d106.sh
clang-11: note: diagnostic msg: 

********************

cupy_tb = <traceback object at 0x7f6ecc1ea5f0>, numpy_error = None, numpy_tb = None, accept_error = ()

    def _check_cupy_numpy_error(self, cupy_error, cupy_tb, numpy_error,
                                numpy_tb, accept_error=False):
        # Skip the test if both raised SkipTest.
        if (isinstance(cupy_error, unittest.SkipTest)
                and isinstance(numpy_error, unittest.SkipTest)):
            if cupy_error.args != numpy_error.args:
                raise AssertionError(
                    'Both numpy and cupy were skipped but with different causes.')
            raise numpy_error  # reraise SkipTest
    
        # For backward compatibility
        if accept_error is True:
            accept_error = Exception
        elif not accept_error:
            accept_error = ()
        # TODO(oktua): expected_regexp like numpy.testing.assert_raises_regex
        if cupy_error is None and numpy_error is None:
            self.fail('Both cupy and numpy are expected to raise errors, but not')
        elif cupy_error is None:
            _fail_test_with_unexpected_errors(
                self,
                'Only numpy raises error\n\n{numpy_tb}{numpy_error}',
                None, None, numpy_error, numpy_tb)
        elif numpy_error is None:
            _fail_test_with_unexpected_errors(
                self,
                'Only cupy raises error\n\n{cupy_tb}{cupy_error}',
>               cupy_error, cupy_tb, None, None)

cupy/testing/helper.py:147: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

testcase = <cupy.testing._bundle.TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'} testMethod=test_searchsorted>
msg_format = 'Only cupy raises error\n\n{cupy_tb}{cupy_error}'
cupy_error = `hipcc` command returns non-zero exit status. 
command: ['hipcc', '--genco', '-I/opt/rocm/include', '-I/home/leo/dev/c...n-f4d106.cu
clang-11: note: diagnostic msg: /tmp/kern-f4d106.sh
clang-11: note: diagnostic msg: 

********************

cupy_tb = <traceback object at 0x7f6ecc1ea5f0>, numpy_error = None, numpy_tb = None

    def _fail_test_with_unexpected_errors(
            testcase, msg_format, cupy_error, cupy_tb, numpy_error, numpy_tb):
        # Fails the test due to unexpected errors raised from the test.
        # msg_format may include format placeholders:
        # '{cupy_error}' '{cupy_tb}' '{numpy_error}' '{numpy_tb}'
    
        msg = msg_format.format(
            cupy_error=''.join(str(cupy_error)),
            cupy_tb=''.join(traceback.format_tb(cupy_tb)),
            numpy_error=''.join(str(numpy_error)),
            numpy_tb=''.join(traceback.format_tb(numpy_tb)))
    
        # Fail the test with the traceback of the error (for pytest --pdb)
        try:
            testcase.fail(msg)
        except AssertionError as e:
>           raise e.with_traceback(cupy_tb or numpy_tb)

cupy/testing/helper.py:116: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <cupy.testing._bundle.TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'} testMethod=test_searchsorted>
impl = <function TestSearchSorted.test_searchsorted at 0x7f6e09220050>, args = ()
kw = {'dtype': <class 'numpy.complex64'>, 'xp': <module 'cupy' from '/home/leo/dev/cupy_rocm35/cupy/__init__.py'>}

    def _call_func(self, impl, args, kw):
        try:
>           result = impl(self, *args, **kw)

cupy/testing/helper.py:23: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <cupy.testing._bundle.TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'} testMethod=test_searchsorted>
xp = <module 'cupy' from '/home/leo/dev/cupy_rocm35/cupy/__init__.py'>, dtype = <class 'numpy.complex64'>

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_array_equal()
    def test_searchsorted(self, xp, dtype):
        x = testing.shaped_arange(self.shape, xp, dtype)
        bins = xp.array(self.bins)
>       y = xp.searchsorted(bins, x, side=self.side)

tests/cupy_tests/sorting_tests/test_search.py:584: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

a = array([ 0,  1,  2,  4, 10])
v = array([ 1. +1.j,  2. +2.j,  3. +3.j,  4. +4.j,  5. +5.j,  6. +6.j,
        7. +7.j,  8. +8.j,  9. +9.j, 10.+10.j], dtype=complex64), side = 'left', sorter = None

    def searchsorted(a, v, side='left', sorter=None):
        """Finds indices where elements should be inserted to maintain order.
    
        Find the indices into a sorted array ``a`` such that,
        if the corresponding elements in ``v`` were inserted before the indices,
        the order of ``a`` would be preserved.
    
        Args:
            a (cupy.ndarray): Input array. If ``sorter`` is ``None``, then
                it must be sorted in ascending order,
                otherwise ``sorter`` must be an array of indices that sort it.
            v (cupy.ndarray): Values to insert into ``a``.
            side : {'left', 'right'}
                If ``left``, return the index of the first suitable location found
                If ``right``, return the last such index.
                If there is no suitable index, return either 0 or length of ``a``.
            sorter : 1-D array_like
                Optional array of integer indices that sort array ``a`` into
                ascending order. They are typically the result of
                :func:`~cupy.argsort`.
    
        Returns:
            cupy.ndarray: Array of insertion points with the same shape as ``v``.
    
        .. note:: When a is not in ascending order, behavior is undefined.
    
        .. seealso:: :func:`numpy.searchsorted`
    
        """
>       return _searchsorted(a, v, side, sorter, True)

cupy/_sorting/search.py:364: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

a = array([ 0.+0.j,  1.+0.j,  2.+0.j,  4.+0.j, 10.+0.j], dtype=complex64)
v = array([ 1. +1.j,  2. +2.j,  3. +3.j,  4. +4.j,  5. +5.j,  6. +6.j,
        7. +7.j,  8. +8.j,  9. +9.j, 10.+10.j], dtype=complex64), side = 'left', sorter = None
assume_increasing = True

    def _searchsorted(a, v, side, sorter, assume_increasing):
        """`assume_increasing` is used in the kernel to
        skip monotonically increasing or decreasing verification
        inside the cuda kernel.
        """
        if not isinstance(a, cupy.ndarray):
            raise NotImplementedError('Only int or ndarray are supported for a')
    
        if not isinstance(v, cupy.ndarray):
            raise NotImplementedError('Only int or ndarray are supported for v')
    
        if a.ndim > 1:
            raise ValueError('object too deep for desired array')
        if a.ndim < 1:
            raise ValueError('object of too small depth for desired array')
        if a.size == 0:
            return cupy.zeros(v.shape, dtype=cupy.int64)
    
        a_iscomplex = a.dtype.kind == 'c'
        v_iscomplex = v.dtype.kind == 'c'
    
        if a_iscomplex and not v_iscomplex:
            v = v.astype(a.dtype)
        elif v_iscomplex and not a_iscomplex:
            a = a.astype(v.dtype)
    
        # Numpy does not check if the array is monotonic inside searchsorted
        # which leds to undefined behavior in such cases.
        if sorter is not None:
            if sorter.dtype.kind not in ('i', 'u'):
                raise TypeError('sorter must be of integer type')
            if sorter.size != a.size:
                raise ValueError('sorter.size must equal a.size')
            a = a.take(sorter)
    
        y = cupy.zeros(v.shape, dtype=cupy.int64)
    
>       _searchsorted_kernel(v, a, a.size, side == 'right', assume_increasing, y)

cupy/_sorting/search.py:404: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   kern = self._get_elementwise_kernel(dev_id, arginfos, type_map)

cupy/core/_kernel.pyx:821: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   kern = _get_elementwise_kernel(

cupy/core/_kernel.pyx:846: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   result = f(*args, **kwargs)

cupy/util.pyx:103: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   return _get_simple_elementwise_kernel(

cupy/core/_kernel.pyx:639: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   cpdef function.Function _get_simple_elementwise_kernel(

cupy/core/_kernel.pyx:37: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   module = compile_with_cache(module_code, options)

cupy/core/_kernel.pyx:60: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

>   return cuda.compile_with_cache(

cupy/core/core.pyx:1876: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '#include <cupy/complex.cuh>\n#include <cupy/carray.cuh>\n#include <cupy/atomics.cuh>\n\n    typedef complex<float> S;...+ 1;\n        } else {\n            right = m;\n        }\n    }\n    y = right;\n    ;\n      }\n      ;\n    }\n    '
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include',), arch = None, cache_dir = None
extra_source = '#pragma once\n\n#include <cupy/complex/complex.h>\n\nusing thrust::complex;\nusing thrust::conj;\nusing thrust::real;...<float> atanh(const complex<float>& z) {\n  return detail::complex::catanhf(z);\n}\n#endif\n\n}  // namespace thrust\n'
backend = 'nvrtc'

    def compile_with_cache(
            source, options=(), arch=None, cache_dir=None, extra_source=None,
            backend='nvrtc', *, enable_cooperative_groups=False,
            name_expressions=None, log_stream=None):
    
        if enable_cooperative_groups:
            if backend != 'nvcc':
                raise ValueError(
                    'Cooperative groups is supported only in NVCC backend.')
            if runtime.is_hip:
                raise ValueError(
                    'Cooperative groups is not supported in HIP.')
    
        # TODO(leofang): handle HIP environment
        if name_expressions is not None and backend != 'nvrtc':
            raise NotImplementedError
    
        # We silently ignore CUPY_CACHE_IN_MEMORY if nvcc is in use, because it
        # must dump files to disk.
        cache_in_memory = (
            _get_bool_env_variable('CUPY_CACHE_IN_MEMORY', False)
            and backend == 'nvrtc')
    
        if runtime.is_hip:
            return _compile_with_cache_hip(
                source, options, arch, cache_dir, extra_source,
>               log_stream)

cupy/cuda/compiler.py:332: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '#include <hip/hip_runtime.h>\n#include <cupy/complex.cuh>\n#include <cupy/carray.cuh>\n#include <cupy/atomics.cuh>\n\...+ 1;\n        } else {\n            right = m;\n        }\n    }\n    y = right;\n    ;\n      }\n      ;\n    }\n    '
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include',), arch = 'gfx906', cache_dir = '/home/leo/.cupy/kernel_cache_rocm35'
extra_source = '#pragma once\n\n#include <cupy/complex/complex.h>\n\nusing thrust::complex;\nusing thrust::conj;\nusing thrust::real;...<float> atanh(const complex<float>& z) {\n  return detail::complex::catanhf(z);\n}\n#endif\n\n}  // namespace thrust\n'
log_stream = None, use_converter = True

    def _compile_with_cache_hip(source, options, arch, cache_dir, extra_source,
                                log_stream=None, use_converter=True):
        global _empty_file_preprocess_cache
    
        if _is_cudadevrt_needed(options):
            raise ValueError('separate compilation is not supported in HIP')
    
        if cache_dir is None:
            cache_dir = get_cache_dir()
        if arch is None:
            arch = os.environ.get('HCC_AMDGPU_TARGET')
            if arch is None:
                raise RuntimeError('HCC_AMDGPU_TARGET is not set')
        if use_converter:
            source = _convert_to_hip_source(source)
    
        env = (arch, options, _get_hipcc_version())
        base = _empty_file_preprocess_cache.get(env, None)
        if base is None:
            # This is checking of HIPCC compiler internal version
            base = _preprocess_hipcc('', options)
            _empty_file_preprocess_cache[env] = base
        key_src = '%s %s %s %s' % (env, base, source, extra_source)
    
        key_src = key_src.encode('utf-8')
        name = '%s.hsaco' % hashlib.md5(key_src).hexdigest()
    
        if not os.path.isdir(cache_dir):
            try:
                os.makedirs(cache_dir)
            except OSError:
                if not os.path.isdir(cache_dir):
                    raise
    
        mod = function.Module()
        # To handle conflicts in concurrent situation, we adopt lock-free method
        # to avoid performance degradation.
        path = os.path.join(cache_dir, name)
        if os.path.exists(path):
            with open(path, 'rb') as f:
                data = f.read()
            if len(data) >= 32:
                hash_value = data[:32]
                binary = data[32:]
                binary_hash = hashlib.md5(binary).hexdigest().encode('ascii')
                if hash_value == binary_hash:
                    mod.load(binary)
                    return mod
    
>       binary = compile_using_hipcc(source, options, arch, log_stream)

cupy/cuda/compiler.py:662: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '#include <hip/hip_runtime.h>\n#include <cupy/complex.cuh>\n#include <cupy/carray.cuh>\n#include <cupy/atomics.cuh>\n\...+ 1;\n        } else {\n            right = m;\n        }\n    }\n    y = right;\n    ;\n      }\n      ;\n    }\n    '
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include',), arch = 'gfx906', log_stream = None

    def compile_using_hipcc(source, options, arch, log_stream=None):
        assert len(arch) > 0
        # pass HCC_AMDGPU_TARGET same as arch
        cmd = ['hipcc', '--genco'] + ['-I/opt/rocm/include'] + list(options)
    
        with tempfile.TemporaryDirectory() as root_dir:
            path = os.path.join(root_dir, 'kern')
            in_path = path + '.cpp'
            out_path = path + '.hsaco'
    
            with open(in_path, 'w') as f:
                f.write(source)
    
            cmd += [in_path, '-o', out_path]
    
            try:
                output = _run_cc(cmd, root_dir, 'hipcc', log_stream)
            except HIPCCException as e:
                cex = CompileException(str(e), source, in_path, options,
                                       'hipcc')
    
                dump = _get_bool_env_variable(
                    'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
                if dump:
                    cex.dump(sys.stderr)
    
>               raise cex
E               AssertionError: Only cupy raises error
E               
E                 File "/home/leo/dev/cupy_rocm35/cupy/testing/helper.py", line 23, in _call_func
E                   result = impl(self, *args, **kw)
E                 File "/home/leo/dev/cupy_rocm35/tests/cupy_tests/sorting_tests/test_search.py", line 584, in test_searchsorted
E                   y = xp.searchsorted(bins, x, side=self.side)
E                 File "/home/leo/dev/cupy_rocm35/cupy/_sorting/search.py", line 364, in searchsorted
E                   return _searchsorted(a, v, side, sorter, True)
E                 File "/home/leo/dev/cupy_rocm35/cupy/_sorting/search.py", line 404, in _searchsorted
E                   _searchsorted_kernel(v, a, a.size, side == 'right', assume_increasing, y)
E                 File "cupy/core/_kernel.pyx", line 821, in cupy.core._kernel.ElementwiseKernel.__call__
E                   kern = self._get_elementwise_kernel(dev_id, arginfos, type_map)
E                 File "cupy/core/_kernel.pyx", line 846, in cupy.core._kernel.ElementwiseKernel._get_elementwise_kernel
E                   kern = _get_elementwise_kernel(
E                 File "cupy/util.pyx", line 103, in cupy.util.memoize.decorator.ret
E                   result = f(*args, **kwargs)
E                 File "cupy/core/_kernel.pyx", line 639, in cupy.core._kernel._get_elementwise_kernel
E                   return _get_simple_elementwise_kernel(
E                 File "cupy/core/_kernel.pyx", line 37, in cupy.core._kernel._get_simple_elementwise_kernel
E                   cpdef function.Function _get_simple_elementwise_kernel(
E                 File "cupy/core/_kernel.pyx", line 60, in cupy.core._kernel._get_simple_elementwise_kernel
E                   module = compile_with_cache(module_code, options)
E                 File "cupy/core/core.pyx", line 1876, in cupy.core.core.compile_with_cache
E                   return cuda.compile_with_cache(
E                 File "/home/leo/dev/cupy_rocm35/cupy/cuda/compiler.py", line 332, in compile_with_cache
E                   log_stream)
E                 File "/home/leo/dev/cupy_rocm35/cupy/cuda/compiler.py", line 662, in _compile_with_cache_hip
E                   binary = compile_using_hipcc(source, options, arch, log_stream)
E                 File "/home/leo/dev/cupy_rocm35/cupy/cuda/compiler.py", line 574, in compile_using_hipcc
E                   raise cex
E               `hipcc` command returns non-zero exit status. 
E               command: ['hipcc', '--genco', '-I/opt/rocm/include', '-I/home/leo/dev/cupy_rocm35/cupy/core/include', '/tmp/tmp496fd4o1/kern.cpp', '-o', '/tmp/tmp496fd4o1/kern.hsaco']
E               return-code: 254
E               stdout/stderr: 
E               In file included from /tmp/tmp496fd4o1/kern.cpp:1:
E               In file included from /opt/rocm/include/hip/hip_runtime.h:56:
E               In file included from /opt/rocm/include/hip/hcc_detail/hip_runtime.h:480:
E               /opt/rocm/include/hip/hcc_detail/math_functions.h:1414:19: warning: duplicate 'static' declaration specifier [-Wduplicate-decl-specifier]
E               __DEVICE__ inline static T min(T arg1, T arg2) {
E                                 ^~~~~~~
E               /opt/rocm/include/hip/hcc_detail/math_functions.h:1419:19: warning: duplicate 'static' declaration specifier [-Wduplicate-decl-specifier]
E               __DEVICE__ inline static T max(T arg1, T arg2) {
E                                 ^~~~~~~
E               2 warnings generated when compiling for gfx906.
E               LLVM ERROR: failed to annotate CFG
E               PLEASE submit a bug report to https://bugs.llvm.org/ and include the crash backtrace.
E               Stack dump:
E               0.	Program arguments: /opt/rocm-3.5.0/llvm/bin/llc /tmp/kern-a82c92-gfx906-optimized-93258b.bc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -filetype=obj -amdgpu-early-inline-all=true -amdgpu-function-calls=false -o /tmp/kern-a82c92-gfx906-917ea5.o 
E               1.	Running pass 'Function Pass Manager' on module '/tmp/kern-a82c92-gfx906-optimized-93258b.bc'.
E               2.	Running pass 'SI annotate control flow' on function '@kernel'
E                #0 0x00000000019765da llvm::sys::PrintStackTrace(llvm::raw_ostream&) (/opt/rocm-3.5.0/llvm/bin/llc+0x19765da)
E                #1 0x000000000197434c llvm::sys::RunSignalHandlers() (/opt/rocm-3.5.0/llvm/bin/llc+0x197434c)
E                #2 0x00000000019744b3 SignalHandler(int) (/opt/rocm-3.5.0/llvm/bin/llc+0x19744b3)
E                #3 0x00007fd7a32ad8a0 __restore_rt (/lib/x86_64-linux-gnu/libpthread.so.0+0x128a0)
E                #4 0x00007fd7a1f5ef47 raise /build/glibc-2ORdQG/glibc-2.27/signal/../sysdeps/unix/sysv/linux/raise.c:51:0
E                #5 0x00007fd7a1f608b1 abort /build/glibc-2ORdQG/glibc-2.27/stdlib/abort.c:81:0
E                #6 0x00000000018fa27e llvm::report_fatal_error(llvm::Twine const&, bool) (/opt/rocm-3.5.0/llvm/bin/llc+0x18fa27e)
E                #7 0x00000000018fa3a8 (/opt/rocm-3.5.0/llvm/bin/llc+0x18fa3a8)
E                #8 0x00000000007c9465 (anonymous namespace)::SIAnnotateControlFlow::runOnFunction(llvm::Function&) (/opt/rocm-3.5.0/llvm/bin/llc+0x7c9465)
E                #9 0x0000000001324cf3 llvm::FPPassManager::runOnFunction(llvm::Function&) (/opt/rocm-3.5.0/llvm/bin/llc+0x1324cf3)
E               #10 0x00000000013253e3 llvm::FPPassManager::runOnModule(llvm::Module&) (/opt/rocm-3.5.0/llvm/bin/llc+0x13253e3)
E               #11 0x000000000132574b llvm::legacy::PassManagerImpl::run(llvm::Module&) (/opt/rocm-3.5.0/llvm/bin/llc+0x132574b)
E               #12 0x00000000006f4d18 compileModule(char**, llvm::LLVMContext&) (.constprop.326) (/opt/rocm-3.5.0/llvm/bin/llc+0x6f4d18)
E               #13 0x0000000000684ee5 main (/opt/rocm-3.5.0/llvm/bin/llc+0x684ee5)
E               #14 0x00007fd7a1f41b97 __libc_start_main /build/glibc-2ORdQG/glibc-2.27/csu/../csu/libc-start.c:344:0
E               #15 0x00000000006eeec9 _start (/opt/rocm-3.5.0/llvm/bin/llc+0x6eeec9)
E               clang-11: error: unable to execute command: Aborted (core dumped)
E               clang-11: error: amdgcn-link command failed due to signal (use -v to see invocation)
E               clang version 11.0.0 (/src/external/llvm-project/clang 6c08b900599eee52e12bce1e76b20dc413ce30e7)
E               Target: x86_64-unknown-linux-gnu
E               Thread model: posix
E               InstalledDir: /opt/rocm-3.5.0/llvm/bin
E               clang-11: note: diagnostic msg: 
E               ********************
E               
E               PLEASE ATTACH THE FOLLOWING FILES TO THE BUG REPORT:
E               Preprocessed source(s) and associated run script(s) are located at:
E               clang-11: note: diagnostic msg: /tmp/kern-f4d106.cu
E               clang-11: note: diagnostic msg: /tmp/kern-f4d106.sh
E               clang-11: note: diagnostic msg: 
E               
E               ********************

cupy/cuda/compiler.py:574: AssertionError

During handling of the above exception, another exception occurred:

self = <cupy.testing._bundle.TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'} testMethod=test_searchsorted>
xp = <module 'cupy' from '/home/leo/dev/cupy_rocm35/cupy/__init__.py'>, dtype = <class 'numpy.complex64'>

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_array_equal()
    def test_searchsorted(self, xp, dtype):
        x = testing.shaped_arange(self.shape, xp, dtype)
        bins = xp.array(self.bins)
>       y = xp.searchsorted(bins, x, side=self.side)

tests/cupy_tests/sorting_tests/test_search.py:584: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cupy/_sorting/search.py:364: in searchsorted
    return _searchsorted(a, v, side, sorter, True)
cupy/_sorting/search.py:404: in _searchsorted
    _searchsorted_kernel(v, a, a.size, side == 'right', assume_increasing, y)
cupy/core/_kernel.pyx:821: in cupy.core._kernel.ElementwiseKernel.__call__
    kern = self._get_elementwise_kernel(dev_id, arginfos, type_map)
cupy/core/_kernel.pyx:846: in cupy.core._kernel.ElementwiseKernel._get_elementwise_kernel
    kern = _get_elementwise_kernel(
cupy/util.pyx:103: in cupy.util.memoize.decorator.ret
    result = f(*args, **kwargs)
cupy/core/_kernel.pyx:639: in cupy.core._kernel._get_elementwise_kernel
    return _get_simple_elementwise_kernel(
cupy/core/_kernel.pyx:37: in cupy.core._kernel._get_simple_elementwise_kernel
    cpdef function.Function _get_simple_elementwise_kernel(
cupy/core/_kernel.pyx:60: in cupy.core._kernel._get_simple_elementwise_kernel
    module = compile_with_cache(module_code, options)
cupy/core/core.pyx:1876: in cupy.core.core.compile_with_cache
    return cuda.compile_with_cache(
cupy/cuda/compiler.py:332: in compile_with_cache
    log_stream)
cupy/cuda/compiler.py:662: in _compile_with_cache_hip
    binary = compile_using_hipcc(source, options, arch, log_stream)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '#include <hip/hip_runtime.h>\n#include <cupy/complex.cuh>\n#include <cupy/carray.cuh>\n#include <cupy/atomics.cuh>\n\...+ 1;\n        } else {\n            right = m;\n        }\n    }\n    y = right;\n    ;\n      }\n      ;\n    }\n    '
options = ('-I/home/leo/dev/cupy_rocm35/cupy/core/include',), arch = 'gfx906', log_stream = None

    def compile_using_hipcc(source, options, arch, log_stream=None):
        assert len(arch) > 0
        # pass HCC_AMDGPU_TARGET same as arch
        cmd = ['hipcc', '--genco'] + ['-I/opt/rocm/include'] + list(options)
    
        with tempfile.TemporaryDirectory() as root_dir:
            path = os.path.join(root_dir, 'kern')
            in_path = path + '.cpp'
            out_path = path + '.hsaco'
    
            with open(in_path, 'w') as f:
                f.write(source)
    
            cmd += [in_path, '-o', out_path]
    
            try:
                output = _run_cc(cmd, root_dir, 'hipcc', log_stream)
            except HIPCCException as e:
                cex = CompileException(str(e), source, in_path, options,
                                       'hipcc')
    
                dump = _get_bool_env_variable(
                    'CUPY_DUMP_CUDA_SOURCE_ON_ERROR', False)
                if dump:
                    cex.dump(sys.stderr)
    
>               raise cex
E               AssertionError: Parameterized test failed.
E               
E               Base test method: TestSearchSorted.test_searchsorted
E               Test parameters:
E                 bins: [0, 1, 2, 4, 10]
E                 shape: (10,)
E                 side: left

cupy/cuda/compiler.py:574: AssertionError
------------------------------------------------------------------------ Captured stdout call -------------------------------------------------------------------------
dtype is <class 'numpy.complex64'>
=============================================================================== PASSES ================================================================================
======================================================================= short test summary info =======================================================================
PASSED tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_0_{bins=[], shape=(), side='left'}::test_searchsorted
PASSED tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_1_{bins=[], shape=(), side='right'}::test_searchsorted
PASSED tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_2_{bins=[], shape=(10,), side='left'}::test_searchsorted
PASSED tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_3_{bins=[], shape=(10,), side='right'}::test_searchsorted
PASSED tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_4_{bins=[], shape=(6, 3, 3), side='left'}::test_searchsorted
PASSED tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_5_{bins=[], shape=(6, 3, 3), side='right'}::test_searchsorted
PASSED tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_6_{bins=[0, 1, 2, 4, 10], shape=(), side='left'}::test_searchsorted
PASSED tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_7_{bins=[0, 1, 2, 4, 10], shape=(), side='right'}::test_searchsorted
FAILED tests/cupy_tests/sorting_tests/test_search.py::TestSearchSorted_param_8_{bins=[0, 1, 2, 4, 10], shape=(10,), side='left'}::test_searchsorted - AssertionError...
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
============================================================= 1 failed, 8 passed, 108 deselected in 2.85s =============================================================
```



Discussion:

Resolving patch:
diff --git a/cupy/_sorting/search.py b/cupy/_sorting/search.py
--- a/cupy/_sorting/search.py
+++ b/cupy/_sorting/search.py
@@ -241,11 +241,30 @@ def argwhere(a):
 '''
 
 
+_hip_preamble = r'''
+#ifdef __HIP_DEVICE_COMPILE__
+  #define no_thread_divergence(do_work, to_return) \
+    if (!is_done) {                                \
+      do_work;                                     \
+      is_done = true;                              \
+    }
+#else
+  #define no_thread_divergence(do_work, to_return) \
+    do_work;                                       \
+    if (to_return) { return; }
+#endif
+'''
+
+
 _searchsorted_kernel = core.ElementwiseKernel(
     'S x, raw T bins, int64 n_bins, bool side_is_right, '
     'bool assume_increassing',
     'int64 y',
     '''
+    #ifdef __HIP_DEVICE_COMPILE__
+    bool is_done = false;
+    #endif
+
     // Array is assumed to be monotonically
     // increasing unless a check is requested with the
     // `assume_increassing = False` parameter.
@@ -271,8 +290,7 @@ def argwhere(a):
                 }
             }
         }
-        y = pos;
-        return;
+        no_thread_divergence( y = pos , true )
     }
 
     bool greater = false;
@@ -282,8 +300,7 @@ def argwhere(a):
         greater = (inc ? x > bins[n_bins-1] : x <= bins[n_bins-1]);
     }
     if (greater) {
-        y = n_bins;
-        return;
+        no_thread_divergence( y = n_bins , true )
     }
 
     long long left = 0;
@@ -294,14 +311,12 @@ def argwhere(a):
             ++left;
         }
         if (left == n_bins) {
-            y = n_bins;
-            return;
+            no_thread_divergence( y = n_bins , true )
         }
         if (side_is_right
                 && !_isnan<T>(bins[n_bins-1]) && !_isnan<S>(x)
                 && bins[n_bins-1] > x) {
-            y = n_bins;
-            return;
+            no_thread_divergence( y = n_bins , true )
         }
     }
 
@@ -320,8 +335,8 @@ def argwhere(a):
             right = m;
         }
     }
-    y = right;
-    ''', preamble=_preamble)
+    no_thread_divergence( y = right , false )
+    ''', preamble=_preamble+_hip_preamble)
 
 
 def searchsorted(a, v, side='left', sorter=None):


Test patch: