Repository: cython/cython

Issue:
Support "const" modifier on fused types
```cython
cdef void c_encode_by_bits(const npy_intp n, const npy_intp m, const long * a, 
                           const long bitshift, integer * out) nogil:
    cdef npy_intp i, j 
    cdef long shift
    cdef integer value

    for i in range(n):
        value = 0
        shift = 0
        for j in range(m):
            value += a[i * m + j] << shift
            shift += bitshift
        out[i] = value
            
cdef void c_decode_by_bits(const npy_intp n, const npy_intp m, 
                           const integer * a, const long bitshift, 
                           long * out) nogil:
    cdef npy_intp i, j, k
    cdef long shift, mask = (1 << bitshift) - 1
    
    for i in range(n):
        shift = 0
        for j in range(m):
            out[i * m + j] = (<long> a[i] >> shift) & mask 
            shift += bitshift
```

```cython
ctypedef fused integer:
    char
    unsigned char
    short
    unsigned short
    int
    unsigned int
    long
    unsigned long
```
```
[1/1] Cythonizing locality/helpers/encode.pyx
Traceback (most recent call last):
  File "build.py", line 99, in <module>
    ext_modules = cythonize(cy_extensions) + c_extensions,
  File "/usr/lib/python3.6/site-packages/Cython/Build/Dependencies.py", line 934, in cythonize
    cythonize_one(*args)
  File "/usr/lib/python3.6/site-packages/Cython/Build/Dependencies.py", line 1039, in cythonize_one
    result = compile([pyx_file], options)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/Main.py", line 686, in compile
    return compile_multiple(source, options)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/Main.py", line 664, in compile_multiple
    result = run_pipeline(source, options, context=context)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/Main.py", line 494, in run_pipeline
    err, enddata = Pipeline.run_pipeline(pipeline, source)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/Pipeline.py", line 340, in run_pipeline
    data = phase(data)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/Pipeline.py", line 53, in generate_pyx_code_stage
    module_node.process_implementation(options, result)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/ModuleNode.py", line 137, in process_implementation
    self.generate_c_code(env, options, result)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/ModuleNode.py", line 365, in generate_c_code
    self.body.generate_function_definitions(env, code)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/Nodes.py", line 436, in generate_function_definitions
    stat.generate_function_definitions(env, code)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/Nodes.py", line 436, in generate_function_definitions
    stat.generate_function_definitions(env, code)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/Nodes.py", line 1754, in generate_function_definitions
    self.generate_function_header(code, with_pymethdef=with_pymethdef)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/Nodes.py", line 2466, in generate_function_header
    arg_decl = arg.declaration_code()
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/PyrexTypes.py", line 3050, in declaration_code
    return self.type.declaration_code(self.cname, for_display)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/PyrexTypes.py", line 2372, in declaration_code
    for_display, dll_linkage, pyrex)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/PyrexTypes.py", line 1475, in declaration_code
    return self.const_base_type.declaration_code("const %s" % entity_code, for_display, dll_linkage, pyrex)
  File "/usr/lib/python3.6/site-packages/Cython/Compiler/PyrexTypes.py", line 1533, in declaration_code
    raise Exception("This may never happen, please report a bug")
```

Replacing "integer" with "long" stops the exception.


Discussion:
I think the problem is the `const integer`. Declaring fused types as `const` after the fact seems unsupported. I would consider that a bug, constifying a fused type usage seems reasonable to me.
Here is a (non-minimal) test that fails to compile for me but should probably be allowed (can go into `tests/run/fused_types.pyx`):
```
cdef fused ints_t:
    int
    long

cdef ints_t cdef_const_fused(const ints_t *i):
    return i[0] + 2

def const_fused(const ints_t i):
    """
    >>> const_fused(5)
    12
    """
    return i + cdef_const_fused(&i)
```

For the record, I hit the exact same error when I tried to use const memoryview (from https://github.com/cython/cython/pull/1869) + fused types. To reproduce:

```py
cdef fused floating1d:
    float[:]
    double[:]

def test(const floating1d arr):
    pass
```

As hinted by https://github.com/cython/cython/issues/1772#issuecomment-316138653, a work-around is to use const inside the fused type definition like this:

```py
cdef fused const_floating1d:
    const float[:]
    const double[:]


def test(const_floating1d arr):
    pass
```

> As hinted by #1772 (comment), a work-around is to use const inside the fused type definition like this:

As noted in #2141 this work-around has some limitations because the const memoryview and the non-const memoryview types are seen as independent so that the function signature will be generated for all possible combination of types. For example:

```python

from cython cimport floating
 
 
 cdef fused const_floating1d:
     const float[:]
     const double[:]
 
 
 def test(const_floating1d ro, floating[:] rw):
     rw[:] = ro
```

```
Error compiling Cython file:
------------------------------------------------------------
...
    const float[:]
    const double[:]


def test(const_floating1d ro, floating[:] rw):
    rw[:] = ro
           ^
------------------------------------------------------------

test.pyx:10:12: Different base types for memoryviews (const float, double)

I have another issue with memoyviews of fused types with `const` types. Here is a small code example:

```
cimport cython
cimport numpy as np
import numpy as np

ctypedef np.int8_t int8
ctypedef np.int16_t int16

ctypedef fused FusedType:
    const int8
    const int16
def func(FusedType[:] array):
    pass
```

The Cython compiler raises this error:

```
Error compiling Cython file:
------------------------------------------------------------
...
    cdef __Pyx_memviewslice memslice
    cdef Py_ssize_t itemsize
    cdef bint dtype_signed
    cdef char kind
    itemsize = -1
    cdef bint ___pyx_const int8_is_signed
                          ^
------------------------------------------------------------

(tree fragment):16:27: Syntax error in C variable declaration
```

This does only happen with `ctypedef` defined types, this problem does not occur with native types (char, short). Furthermore, this problem also diappears when a simple variable, rather than a memoryview, is used. 

The error message suggests that `const int8` is treated entirely as type name, which is clearly wrong, since `const` is a modifier.

I am using Cython version 0.28.
It looks like this is not difficult but a bit of work. Basically, in `FusedNode.py`, the `fused_base_types` must be generated without `const` (i.e. `if type.is_const: type = type.const_base_type`), but then the specialisation afterwards must look into `const` types to see if they are fused, and then specialise the type into the corresponding const type again. Not sure if this can be done locally in `PyrexTypes.py` or if it requires some non-local (but probably repetitive) code changes.
PR welcome.
One challenge with the workaround is declaring an array or value within the function that matches the fused type without being constant.
While playing with a workaround similar to @lesteve's, came up with the following tweak.

To ensure interoperability between the two types, we can add an `if` check that compares the two types. Interestingly the `const_floating` will drop the `const` part of its definition here. So a direct `is`/`is not` comparison works.

All that is left is to bail out of bad prototypes preferably with some error that makes the user aware. To that extent, we raise a `TypeError` explaining the type mismatch to the user. Cython also recognizes the rest of the function is unreachable so does not generate the rest of it (avoiding any compile errors on the way).

Only downside is Cython will warn the rest of the function is unreachable. We can fix this by using a decorator on the function to disable the `unreachable` warning. Though this will suppress any other such warnings in the function. The other alternatively would be to move the rest of the code into an `else` branch of type check condition. Maybe other options exist here?

This is pretty similar to how [`static_assert`]( https://en.cppreference.com/w/cpp/language/static_assert ) works in C++. So this is a common way of solving this sort of problem. Maybe Cython can even leverage it under the hood?

```cython
cimport cython
from cython cimport floating


cdef fused const_floating:
    const float
    const double


@cython.warn.unreachable(False)
cpdef void my_safe_float_copy(floating[:] a, const_floating[:] b):
    if floating is not const_floating:
        raise TypeError("Types of a and b don't match")

    a[:] = b.copy()
```
> Maybe Cython can even leverage it under the hood?

No, we should avoid generating the functions entirely. My comment above explains what needs to be done.
We spend half a day investigating this issue with @lesteve at EuroScipy. Posting a (partial) status update in case it might be helpful.

A (possibly) minimal unit test that fails can be found [in this branch](https://github.com/cython/cython/compare/master...lesteve:test-const-fused-type-memoryview), where to produce an error it is sufficient to run,
```py
python -m Cython.Build.Cythonize cython/tests/memoryview/const_fused_type_memoryview.pyx
```
or alternatively,
```
python runtests.py -T 1772
```
(though the traceback in the latter case is less detailed).

Below are a few additional observation (from someone with no formal CS background or in depth understanding how Cython works, and possibly I missed some contribution docs),
 - Looking at the generated C code,
     * the difference between `float` and `const float` declaration is just a few additional "const" declaration
     * the difference between `float` and `floating` (fused float type) is fairly large (though this has been tested with memoryviews: with plain scalars, it might not longer be the case)

   so the probable expected result of the PR that would fix this issue might be a slight modification of the resulting C code.
 - in the overall [processing pipeline](https://github.com/cython/cython/blob/fd36ee4ab0dec15fcaec42bc0e72b30a60c27d50/Cython/Compiler/Pipeline.py#L181) the processing step which is probably affected is `AnalyseDeclarationsTransform` (cf @scoder's above comment https://github.com/cython/cython/issues/1772#issuecomment-376629824).
  - we had some trouble specifying breakpoints for pdb in the code as it appears that some source files are cythonized. Using `git clean -xdf` could be a possible workaround.

There are probably much better ways of approaching this issue for someone with better knowledge of Cython code base.

Resolving patch:
diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py
--- a/Cython/Compiler/PyrexTypes.py
+++ b/Cython/Compiler/PyrexTypes.py
@@ -1573,6 +1573,8 @@ def __hash__(self):
 class CConstOrVolatileType(BaseType):
     "A C const or volatile type"
 
+    subtypes = ['cv_base_type']
+
     is_cv_qualified = 1
 
     def __init__(self, base_type, is_const=0, is_volatile=0):


Test patch:
diff --git a/tests/errors/fused_types.pyx b/tests/errors/fused_types.pyx
--- a/tests/errors/fused_types.pyx
+++ b/tests/errors/fused_types.pyx
@@ -1,4 +1,5 @@
 # mode: error
+# ticket: 1772
 
 cimport cython
 from cython import fused_type
@@ -64,17 +65,30 @@ ctypedef fused fused2:
 func(x, y)
 
 
+cdef fused mix_const_t:
+    int
+    const int
+
+cdef cdef_func_with_mix_const_type(mix_const_t val):
+    print(val)
+
+# Mixing const and non-const type makes fused type ambiguous
+cdef_func_with_mix_const_type(1)
+
+
 _ERRORS = u"""
-10:15: fused_type does not take keyword arguments
-15:33: Type specified multiple times
-26:0: Invalid use of fused types, type cannot be specialized
-26:4: Not enough types specified to specialize the function, int2_t is still fused
+11:15: fused_type does not take keyword arguments
+16:33: Type specified multiple times
 27:0: Invalid use of fused types, type cannot be specialized
 27:4: Not enough types specified to specialize the function, int2_t is still fused
-28:16: Call with wrong number of arguments (expected 2, got 1)
-29:16: Call with wrong number of arguments (expected 2, got 3)
-36:6: Invalid base type for memoryview slice: int *
-39:0: Fused lambdas not allowed
-42:5: Fused types not allowed here
-45:9: Fused types not allowed here
+28:0: Invalid use of fused types, type cannot be specialized
+28:4: Not enough types specified to specialize the function, int2_t is still fused
+29:16: Call with wrong number of arguments (expected 2, got 1)
+30:16: Call with wrong number of arguments (expected 2, got 3)
+37:6: Invalid base type for memoryview slice: int *
+40:0: Fused lambdas not allowed
+43:5: Fused types not allowed here
+46:9: Fused types not allowed here
+76:0: Invalid use of fused types, type cannot be specialized
+76:29: ambiguous overloaded method
 """
diff --git a/tests/memoryview/numpy_memoryview_readonly.pyx b/tests/memoryview/numpy_memoryview_readonly.pyx
--- a/tests/memoryview/numpy_memoryview_readonly.pyx
+++ b/tests/memoryview/numpy_memoryview_readonly.pyx
@@ -1,10 +1,14 @@
 # mode: run
 # tag: readonly, const, numpy
+# ticket: 1772
 
 import numpy as np
+cimport cython
 
-def new_array():
-    return np.arange(10).astype('float')
+def new_array(dtype='float', writeable=True):
+    array = np.arange(10, dtype=dtype)
+    array.setflags(write=writeable)
+    return array
 
 ARRAY = new_array()
 
@@ -124,3 +128,45 @@ def test_copy():
     rw[1] = 2
     rw2[2] = 2
     return rw[0], rw[1], rw[2], rw2[0], rw2[1], rw2[2]
+
+
+cdef getmax_floating(const cython.floating[:] x):
+    """Function with fused type, should work with both ro and rw memoryviews"""
+    cdef cython.floating max_val = - float('inf')
+    for val in x:
+        if val > max_val:
+            max_val = val
+    return max_val
+
+
+def test_mmview_const_fused_cdef():
+    """Test cdef function with const fused type memory view as argument.
+
+    >>> test_mmview_const_fused_cdef()
+    """
+    cdef float[:] data_rw = new_array(dtype='float32')
+    assert getmax_floating(data_rw) == 9
+
+    cdef const float[:] data_ro = new_array(dtype='float32', writeable=False)
+    assert getmax_floating(data_ro) == 9
+
+
+def test_mmview_const_fused_def(const cython.floating[:] x):
+    """Test def function with const fused type memory view as argument.
+
+    With read-write numpy array:
+
+    >>> test_mmview_const_fused_def(new_array('float32', writeable=True))
+    0.0
+    >>> test_mmview_const_fused_def(new_array('float64', writeable=True))
+    0.0
+
+    With read-only numpy array:
+
+    >>> test_mmview_const_fused_def(new_array('float32', writeable=False))
+    0.0
+    >>> test_mmview_const_fused_def(new_array('float64', writeable=False))
+    0.0
+    """
+    cdef cython.floating result = x[0]
+    return result
diff --git a/tests/run/fused_types.pyx b/tests/run/fused_types.pyx
--- a/tests/run/fused_types.pyx
+++ b/tests/run/fused_types.pyx
@@ -1,4 +1,5 @@
 # mode: run
+# ticket: 1772
 
 cimport cython
 from cython.view cimport array
@@ -361,6 +362,22 @@ def test_fused_memslice_dtype_repeated_2(cython.floating[:] array1, cython.float
     """
     print cython.typeof(array1), cython.typeof(array2), cython.typeof(array3)
 
+def test_fused_const_memslice_dtype_repeated(const cython.floating[:] array1, cython.floating[:] array2):
+    """Test fused types memory view with one being const
+
+    >>> sorted(test_fused_const_memslice_dtype_repeated.__signatures__)
+    ['double', 'float']
+
+    >>> test_fused_const_memslice_dtype_repeated(get_array(8, 'd'), get_array(8, 'd'))
+    const double[:] double[:]
+    >>> test_fused_const_memslice_dtype_repeated(get_array(4, 'f'), get_array(4, 'f'))
+    const float[:] float[:]
+    >>> test_fused_const_memslice_dtype_repeated(get_array(8, 'd'), get_array(4, 'f'))
+    Traceback (most recent call last):
+    ValueError: Buffer dtype mismatch, expected 'double' but got 'float'
+    """
+    print cython.typeof(array1), cython.typeof(array2)
+
 def test_cython_numeric(cython.numeric arg):
     """
     Test to see whether complex numbers have their utility code declared
@@ -386,6 +403,18 @@ def test_index_fused_args(cython.floating f, ints_t i):
     """
     _test_index_fused_args[cython.floating, ints_t](f, i)
 
+cdef _test_index_const_fused_args(const cython.floating f, const ints_t i):
+    print(cython.typeof(f), cython.typeof(i))
+
+def test_index_const_fused_args(const cython.floating f, const ints_t i):
+    """Test indexing function implementation with const fused type args
+
+    >>> import cython
+    >>> test_index_const_fused_args[cython.double, cython.int](2.0, 3)
+    ('const double', 'const int')
+    """
+    _test_index_const_fused_args[cython.floating, ints_t](f, i)
+
 
 def test_composite(fused_composite x):
     """
@@ -400,3 +429,27 @@ def test_composite(fused_composite x):
         return x
     else:
         return 2 * x
+
+
+cdef cdef_func_const_fused_arg(const cython.floating val,
+                               const fused_type1 * ptr_to_const,
+                               const (cython.floating *) const_ptr):
+    print(val, cython.typeof(val))
+    print(ptr_to_const[0], cython.typeof(ptr_to_const[0]))
+    print(const_ptr[0], cython.typeof(const_ptr[0]))
+
+    ptr_to_const = NULL  # pointer is not const, value is const
+    const_ptr[0] = 0.0  # pointer is const, value is not const
+
+def test_cdef_func_with_const_fused_arg():
+    """Test cdef function with const fused type argument
+
+    >>> test_cdef_func_with_const_fused_arg()
+    (0.0, 'const float')
+    (1, 'const int')
+    (2.0, 'float')
+    """
+    cdef float arg0 = 0.0
+    cdef int arg1 = 1
+    cdef float arg2 = 2.0
+    cdef_func_const_fused_arg(arg0, &arg1, &arg2)