Repository: numba/numba

Issue:
JIT compiler fails on code containing non-latin1 characters
JIT compiler fails on any function containing any non-latin1 characters.

Versions:
Python 3.4
numba 0.26.0
llvmlite 0.12.0

I am not totally sure whether this isn't more of a llvmlite issue.

A minimal example:

```
from numba import jit

@jit
def fun():
    č = 1
    return č

fun()
```

which yields

```
Traceback (most recent call last):
  File "numbafail.py", line 8, in <module>
    fun()
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/dispatcher.py", line 277, in _compile_for_args
    return self.compile(tuple(real_args))
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/dispatcher.py", line 495, in compile
    cres = self._compiler.compile(args, return_type)
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/dispatcher.py", line 76, in compile
    flags=flags, locals=self.locals)
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 696, in compile_extra
    return pipeline.compile_extra(func)
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 369, in compile_extra
    return self.compile_bytecode(bc, func_attr=self.func_attr)
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 378, in compile_bytecode
    return self._compile_bytecode()
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 662, in _compile_bytecode
    res = pm.run(self.status)
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 257, in run
    raise patched_exception
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 249, in run
    stage()
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 585, in stage_objectmode_backend
    self._backend(lowerfn, objectmode=True)
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 562, in _backend
    lowered = lowerfn()
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 536, in backend_object_mode
    self.flags)
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/compiler.py", line 848, in py_lowering_stage
    lower.lower()
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/lowering.py", line 142, in lower
    self.library.add_ir_module(self.module)
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/numba/targets/codegen.py", line 146, in add_ir_module
    ll_module = ll.parse_assembly(str(ir_module))
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/llvmlite/binding/module.py", line 16, in parse_assembly
    llvmir = _encode_string(llvmir)
  File "/u/46/necadam1/unix/.local/lib/python3.4/site-packages/llvmlite/binding/common.py", line 17, in _encode_string
    encoded = s.encode('latin1')
```



Discussion:
At its core, this is an LLVM issue that we might be able to hide (eventually).  The current LLVM IR only supports ASCII identifiers:

http://llvm.org/docs/LangRef.html#identifiers

In order to handle Python unicode identifiers in llvmlite, we would need to come up with an identifier encoding to map them to ASCII during code generation.

I see, but then I still don't understand why everything seems to work if I use characters from the upper half of latin1 (i.e. stil non-ASCII) instead.
Anyway, so would there be anything wrong about implementing some arbitrary meaningful mapping from unicode to ASCII strings in [llvmlite/llvmlite/binding/common.py](https://github.com/numba/llvmlite/blob/master/llvmlite/binding/common.py)? Or am I missing something?

Quoting from the LLVM docs:

> Identifiers that require other characters in their names can be surrounded with quotes. Special characters may be escaped using "\xx" where xx is the ASCII code for the character in hexadecimal. In this way, any character can be used in a name value, even quotes themselves.

The problem is they don't say what a "character" is. ASCII? Latin-1? Unicode? Who knows. They probably treat IR as a stream of bytes, following the broken C model of "text" strings.

Still, Numba could probably escape any non-ASCII characters in user identifiers, since they are not meant to be visible from the outside (except perhaps for `@cfunc`).


Resolving patch:
diff --git a/numba/cgutils.py b/numba/cgutils.py
--- a/numba/cgutils.py
+++ b/numba/cgutils.py
@@ -7,7 +7,6 @@
 import collections
 from contextlib import contextmanager
 import functools
-import re
 
 from llvmlite import ir
 
@@ -967,3 +966,19 @@ def printf(builder, format, *args):
     # Call
     ptr_fmt = builder.bitcast(global_fmt, cstring)
     return builder.call(fn, [ptr_fmt] + list(args))
+
+
+if utils.PY3:
+    def normalize_ir_text(text):
+        """
+        Normalize the given string to latin1 compatible encoding that is suitable
+        for use in LLVM IR.
+        """
+        # Just re-encoding to latin1 is enough
+        return text.encode('utf8').decode('latin1')
+else:
+    def normalize_ir_text(text):
+        """
+        No-op for python2. Assume there won't be unicode names.
+        """
+        return text
diff --git a/numba/funcdesc.py b/numba/funcdesc.py
--- a/numba/funcdesc.py
+++ b/numba/funcdesc.py
@@ -9,6 +9,7 @@
 from types import ModuleType
 
 from . import six, types
+from .utils import PY3
 
 
 def transform_arg_name(arg):
@@ -25,9 +26,13 @@ def transform_arg_name(arg):
 
 def default_mangler(name, argtypes):
     codedargs = '.'.join(transform_arg_name(a).replace(' ', '_')
-                             for a in argtypes)
-    return '.'.join([name, codedargs])
-
+                         for a in argtypes)
+    fullname = '.'.join([name, codedargs])
+    out = fullname.encode('ascii', 'backslashreplace')
+    # for py3, convert bytes back to  str
+    if PY3:
+        out = out.decode('ascii')
+    return out
 
 # A dummy module for dynamically-generated functions
 _dynamic_modname = '<dynamic>'
@@ -75,7 +80,7 @@ def __init__(self, native, modname, qualname, unique_name, doc,
         if self.modname:
             # XXX choose a different convention for object mode
             self.mangled_name = mangler('%s.%s' % (self.modname, self.unique_name),
-                                        self.argtypes)
+                                                   self.argtypes)
         else:
             self.mangled_name = mangler(self.unique_name, self.argtypes)
         self.inline = inline
diff --git a/numba/targets/codegen.py b/numba/targets/codegen.py
--- a/numba/targets/codegen.py
+++ b/numba/targets/codegen.py
@@ -9,7 +9,7 @@
 import llvmlite.binding as ll
 import llvmlite.ir as llvmir
 
-from numba import config, utils
+from numba import config, utils, cgutils
 from numba.runtime.atomicops import remove_redundant_nrt_refct
 
 _x86arch = frozenset(['x86', 'i386', 'i486', 'i586', 'i686', 'i786',
@@ -42,7 +42,7 @@ def __init__(self, codegen, name):
         self._linking_libraries = set()
         self._final_module = ll.parse_assembly(
             str(self._codegen._create_empty_module(self._name)))
-        self._final_module.name = self._name
+        self._final_module.name = cgutils.normalize_ir_text(self._name)
         # Remember this on the module, for the object cache hooks
         self._final_module.__library = weakref.proxy(self)
         self._shared_module = None
@@ -143,7 +143,8 @@ def add_ir_module(self, ir_module):
         """
         self._raise_if_finalized()
         assert isinstance(ir_module, llvmir.Module)
-        ll_module = ll.parse_assembly(str(ir_module))
+        ir = cgutils.normalize_ir_text(str(ir_module))
+        ll_module = ll.parse_assembly(ir)
         ll_module.name = ir_module.name
         ll_module.verify()
         self.add_llvm_module(ll_module)
@@ -422,7 +423,7 @@ def _init(self, llvm_module):
                                       self._library_class._object_getbuffer_hook)
 
     def _create_empty_module(self, name):
-        ir_module = lc.Module(name)
+        ir_module = lc.Module(cgutils.normalize_ir_text(name))
         ir_module.triple = ll.get_process_triple()
         if self._data_layout:
             ir_module.data_layout = self._data_layout


Test patch:
diff --git a/numba/tests/test_mangling.py b/numba/tests/test_mangling.py
new file mode 100644
--- /dev/null
+++ b/numba/tests/test_mangling.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+Test function name mangling.
+The mangling affects the ABI of numba compiled binaries.
+"""
+
+from numba import types
+from numba.funcdesc import default_mangler
+from .support import unittest, TestCase
+
+
+class TestMangling(TestCase):
+    def test_one_args(self):
+        fname = 'foo'
+        argtypes = types.int32,
+        name = default_mangler(fname, argtypes)
+        self.assertEqual(name, 'foo.int32')
+
+    def test_two_args(self):
+        fname = 'foo'
+        argtypes = types.int32, types.float32
+        name = default_mangler(fname, argtypes)
+        self.assertEqual(name, 'foo.int32.float32')
+
+    def test_unicode_fname(self):
+        fname = u'foಠ'
+        argtypes = types.int32, types.float32
+        name = default_mangler(fname, argtypes)
+        self.assertIsInstance(name, str)
+        expect = 'fo\\u{0:04x}.int32.float32'.format(ord(u'ಠ'))
+        self.assertEqual(name, expect)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/numba/tests/test_unicode_names.py b/numba/tests/test_unicode_names.py
new file mode 100644
--- /dev/null
+++ b/numba/tests/test_unicode_names.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function, absolute_import
+
+
+from numba import njit, cfunc, cgutils
+from numba.six import exec_
+from numba.utils import PY2
+
+from .support import TestCase, unittest
+
+unicode_name1 = u"""
+def unicode_name1(ಠ_ರೃ, ಠਊಠ):
+    return (ಠ_ರೃ) + (ಠਊಠ)
+"""
+
+unicode_name2 = u"""
+def Ծ_Ծ(ಠ_ರೃ, ಠਊಠ):
+    return (ಠ_ರೃ) + (ಠਊಠ)
+"""
+
+
+@unittest.skipIf(PY2, "unicode identifier not supported in python2")
+class TestUnicodeNames(TestCase):
+    def make_testcase(self, src, fname):
+        glb = {}
+        exec_(src, glb)
+        fn = glb[fname]
+        return fn
+
+    def test_unicode_name1(self):
+        fn = self.make_testcase(unicode_name1, 'unicode_name1')
+        cfn = njit(fn)
+        self.assertEqual(cfn(1, 2), 3)
+
+    def test_unicode_name2(self):
+        fn = self.make_testcase(unicode_name2, 'Ծ_Ծ')
+        cfn = njit(fn)
+        self.assertEqual(cfn(1, 2), 3)
+
+    def test_cfunc(self):
+        fn = self.make_testcase(unicode_name2, 'Ծ_Ծ')
+        cfn = cfunc("int32(int32, int32)")(fn)
+        self.assertEqual(cfn.ctypes(1, 2), 3)
+
+
+class TestUnicodeUtils(TestCase):
+    def test_normalize_ir_text(self):
+        # non-unicode input
+        out = cgutils.normalize_ir_text('abc')
+        # str returned
+        self.assertIsInstance(out, str)
+        # try encoding to latin
+        out.encode('latin1')
+
+    @unittest.skipIf(PY2, "unicode identifier not supported in python2")
+    def test_normalize_ir_text_py3(self):
+        # unicode input
+        out = cgutils.normalize_ir_text(unicode_name2)
+        # str returned
+        self.assertIsInstance(out, str)
+        # try encoding to latin
+        out.encode('latin1')
+
+
+if __name__ == '__main__':
+    unittest.main()
+