Repository: numba/numba

Issue:
Cannot lower constant bytes.
<!--

Thanks for opening an issue! To help the Numba team handle your information
efficiently, please first ensure that there is no other issue present that
already describes the issue you have
(search at https://github.com/numba/numba/issues?&q=is%3Aissue).

For more general "how do I do X?" type questions, please speak to us in real
time on https://gitter.im/numba/numba or post to the Numba mailing list
https://groups.google.com/a/continuum.io/forum/#!forum/numba-users.

-->

## Reporting a bug

<!--

Before submitting a bug report please ensure that you can check off these boxes:

-->

- [ ] I am using the latest released version of Numba (most recent is visible in
 the change log (https://github.com/numba/numba/blob/master/CHANGE_LOG).
- [x] I have included below a minimal working reproducer (if you are unsure how
 to write one see http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports).

<!--

Please include details of the bug here, including, if applicable, what you
expected to happen!

-->

Running
```
import numpy as np
from numba import njit
@njit
def foo():
    np.frombuffer(b'foo')
foo()
```
triggers internal error:
```
Traceback (most recent call last):
  File "/home/pearu/git/Quansight/numba/numba/targets/base.py", line 499, in get_constant_generic
    impl = self._get_constants.find((ty,))
  File "/home/pearu/git/Quansight/numba/numba/targets/base.py", line 50, in find
    out = self._find(sig)
  File "/home/pearu/git/Quansight/numba/numba/targets/base.py", line 59, in _find
    raise NotImplementedError(self, sig)
NotImplementedError: (<numba.targets.base.OverloadSelector object at 0x7fdba6f28860>, (readonly bytes(uint8, 1d, C),))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/pearu/git/Quansight/numba/numba/errors.py", line 662, in new_error_context
    yield
  File "/home/pearu/git/Quansight/numba/numba/lowering.py", line 258, in lower_block
    self.lower_inst(inst)
  File "/home/pearu/git/Quansight/numba/numba/lowering.py", line 301, in lower_inst
    val = self.lower_assign(ty, inst)
  File "/home/pearu/git/Quansight/numba/numba/lowering.py", line 454, in lower_assign
    value.value)
  File "/home/pearu/git/Quansight/numba/numba/targets/base.py", line 502, in get_constant_generic
    raise NotImplementedError("Cannot lower constant of type '%s'" % (ty,))
NotImplementedError: Cannot lower constant of type 'readonly bytes(uint8, 1d, C)'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "bug_frombuffer_bytes.py", line 6, in <module>
    foo()
  File "/home/pearu/git/Quansight/numba/numba/dispatcher.py", line 395, in _compile_for_args
    raise e
  File "/home/pearu/git/Quansight/numba/numba/dispatcher.py", line 352, in _compile_for_args
    return self.compile(tuple(argtypes))
  File "/home/pearu/git/Quansight/numba/numba/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/home/pearu/git/Quansight/numba/numba/dispatcher.py", line 693, in compile
    cres = self._compiler.compile(args, return_type)
  File "/home/pearu/git/Quansight/numba/numba/dispatcher.py", line 76, in compile
    status, retval = self._compile_cached(args, return_type)
  File "/home/pearu/git/Quansight/numba/numba/dispatcher.py", line 90, in _compile_cached
    retval = self._compile_core(args, return_type)
  File "/home/pearu/git/Quansight/numba/numba/dispatcher.py", line 108, in _compile_core
    pipeline_class=self.pipeline_class)
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 972, in compile_extra
    return pipeline.compile_extra(func)
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 390, in compile_extra
    return self._compile_bytecode()
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 903, in _compile_bytecode
    return self._compile_core()
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 890, in _compile_core
    res = pm.run(self.status)
  File "/home/pearu/git/Quansight/numba/numba/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 266, in run
    raise patched_exception
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 257, in run
    stage()
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 764, in stage_nopython_backend
    self._backend(lowerfn, objectmode=False)
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 703, in _backend
    lowered = lowerfn()
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 690, in backend_nopython_mode
    self.metadata)
  File "/home/pearu/git/Quansight/numba/numba/compiler.py", line 1143, in native_lowering_stage
    lower.lower()
  File "/home/pearu/git/Quansight/numba/numba/lowering.py", line 177, in lower
    self.lower_normal_function(self.fndesc)
  File "/home/pearu/git/Quansight/numba/numba/lowering.py", line 218, in lower_normal_function
    entry_block_tail = self.lower_function_body()
  File "/home/pearu/git/Quansight/numba/numba/lowering.py", line 243, in lower_function_body
    self.lower_block(block)
  File "/home/pearu/git/Quansight/numba/numba/lowering.py", line 258, in lower_block
    self.lower_inst(inst)
  File "/home/pearu/mconda3/envs/numba-dev/lib/python3.7/contextlib.py", line 130, in __exit__
    self.gen.throw(type, value, traceback)
  File "/home/pearu/git/Quansight/numba/numba/errors.py", line 670, in new_error_context
    six.reraise(type(newerr), newerr, tb)
  File "/home/pearu/git/Quansight/numba/numba/six.py", line 659, in reraise
    raise value
numba.errors.LoweringError: Failed in nopython mode pipeline (step: nopython mode backend)
Cannot lower constant of type 'readonly bytes(uint8, 1d, C)'

File "bug_frombuffer_bytes.py", line 5:
def foo():
    np.frombuffer(b'foo')
    ^

[1] During: lowering "$const0.3 = const(bytes, b'foo')" at bug_frombuffer_bytes.py (5)
-------------------------------------------------------------------------------
This should not have happened, a problem has occurred in Numba's internals.
You are currently using Numba version 0.46.0dev0+42.g81b91f93c.

<snip>
```


Discussion:
Thanks for the report, I can reproduce. I don't think that this is a direct issue with `np.frombuffer` but rather one of bytes not being a lowerable constant type, this has the same problem with no NumPy involved:

```python
from numba import njit

@njit
def foo():
    x = b'foo'

foo()
```


Had a quick look at this, it seems like its relatively easy to compile in some const bytes. The difficulty arises when you want to do something with them... so, what is it you are wanting to do with them @pearu, just `np.frombuffer`? If so, I have the following working:
```python
import numpy as np
from numba import njit

@njit
def foo():
    myarray = np.frombuffer(b'foo', dtype=np.uint8)
    print(myarray)

foo()
```
and can submit a patch.
Background: this is related to issue #4018. I now have a workaround for it but resolving this issue might simplify the current workaround.

The basic idea of the workaround is that in order to compare arrays containing str values, the arrays are viewed as np.uint8 arrays for which `(x == y).all()` works.

Here's an example that does not work:
```
import numpy as np
@njit
def foo(x):
    xx = np.frombuffer(x, np.uint8)
    yy = np.frombuffer(b'foo', np.uint8)
    return (xx == yy).all()
foo(y)
```
but has the following workaround:
```
import numpy as np
yy = np.frombuffer(b'foo', np.uint8)
@njit
def foo(x):
    xx = np.frombuffer(x, np.uint8)
    return (xx == yy).all()
foo(y)
```

Resolving patch:
diff --git a/numba/targets/arrayobj.py b/numba/targets/arrayobj.py
--- a/numba/targets/arrayobj.py
+++ b/numba/targets/arrayobj.py
@@ -2393,9 +2393,18 @@ def constant_record(context, builder, ty, pyval):
     return cgutils.alloca_once_value(builder, val)
 
 
+@lower_constant(types.Bytes)
+def constant_bytes(context, builder, ty, pyval):
+    """
+    Create a constant array from bytes (mechanism is target-dependent).
+    """
+    buf = np.array(bytearray(pyval), dtype=np.uint8)
+    return context.make_constant_array(builder, ty, buf)
+
 #-------------------------------------------------------------------------------
 # Comparisons
 
+
 @lower_builtin(operator.is_, types.Array, types.Array)
 def array_is(context, builder, sig, args):
     aty, bty = sig.args


Test patch:
diff --git a/numba/tests/test_array_constants.py b/numba/tests/test_array_constants.py
--- a/numba/tests/test_array_constants.py
+++ b/numba/tests/test_array_constants.py
@@ -6,6 +6,7 @@
 from numba.compiler import compile_isolated
 from numba.errors import TypingError
 from numba import jit, types, typeof
+from numba.utils import IS_PY3
 
 
 a0 = np.array(42)
@@ -25,6 +26,9 @@
 a6 = np.frombuffer(b"XXXX_array_contents_XXXX", dtype=np.float32)
 
 
+myarray = np.array([1, ])
+
+
 def getitem0(i):
     return a0[()]
 
@@ -61,6 +65,10 @@ def write_to_global_array():
     myarray[0] = 1
 
 
+def bytes_as_const_array():
+    return np.frombuffer(b'foo', dtype=np.uint8)
+
+
 class TestConstantArray(unittest.TestCase):
     """
     Test array constants.
@@ -156,7 +164,7 @@ def pyfunc():
             self.assertIs(biggie, out)
             # Remove all local references to biggie
             del out
-            biggie = None  #  del biggie is syntax error in py2
+            biggie = None  # del biggie is syntax error in py2
             # Run again and verify result
             out = cres.entry_point()
             np.testing.assert_equal(expect, out)
@@ -174,5 +182,15 @@ def pyfunc():
         test(f_array)
 
 
+@unittest.skipUnless(IS_PY3, "Python 3 only")
+class TestConstantBytes(unittest.TestCase):
+    def test_constant_bytes(self):
+        pyfunc = bytes_as_const_array
+        cres = compile_isolated(pyfunc, ())
+        cfunc = cres.entry_point
+
+        np.testing.assert_array_equal(pyfunc(), cfunc())
+
+
 if __name__ == '__main__':
     unittest.main()