Repository: cython/cython

Issue:
Imports are slower than with CPython
If I compile the following functions using Cython:
```python
def _noop_bench():
    pass

def _import_bench():
    import pyarrow.pandas_compat as pdcompat
```

And then measure their performance:
```python
>>> %timeit lib._noop_bench()                                                                                                                                     
66.3 ns ± 0.379 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
>>> %timeit lib._import_bench()                                                                                                                                   
987 ns ± 21.1 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
>>> %timeit lib._import_bench()                                                                                                                                   
966 ns ± 17.6 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
```

The import overhead (~ 900 ns. per import statement) appears much slower than what CPython achieves:
```python
>>> %timeit import pyarrow.pandas_compat as pdcompat                                                                                                              
253 ns ± 4.52 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
>>> %timeit import pyarrow.pandas_compat as pdcompat                                                                                                              
254 ns ± 3.66 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)
```

This is with Python 3, Cython 0.29.4 and `language_level = 3`.


Discussion:
Hmm, interesting. My first thought was Py2-style imports, but you're using `language_level=3`, so that's not it.

Importing has never really been a priority for optimisation, more of a "just make it work across all Python versions" kind of task. But I can't see a reason why Cython should be slower than CPython here, let alone 4x as slow. Definitely worth investigating.
I investigated this. CPython imports the module by dotted name and then looks up attributes on the result until it found the final imported sub-sub-sub-…module.  Cython (and probably Pyrex already) took the easy path of passing a non-empty name list into the import function (actually the list `['*']`), which then takes that as a signal to return the imported submodule instead of the top-level module. That is much slower. I'll see what I can do about it.

Resolving patch:
diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py
--- a/Cython/Compiler/ExprNodes.py
+++ b/Cython/Compiler/ExprNodes.py
@@ -2527,44 +2527,61 @@ class ImportNode(ExprNode):
     #                           relative to the current module.
     #                     None: decide the level according to language level and
     #                           directives
+    #  get_top_level_module   int          true: return top-level module, false: return imported module
+    #  module_names           TupleNode    the separate names of the module and submodules, or None
 
     type = py_object_type
+    module_names = None
+    get_top_level_module = False
+    is_temp = True
 
-    subexprs = ['module_name', 'name_list']
+    subexprs = ['module_name', 'name_list', 'module_names']
 
     def analyse_types(self, env):
         if self.level is None:
-            if (env.directives['py2_import'] or
-                Future.absolute_import not in env.global_scope().context.future_directives):
+            # For modules in packages, and without 'absolute_import' enabled, try relative (Py2) import first.
+            if env.global_scope().parent_module and (
+                    env.directives['py2_import'] or
+                    Future.absolute_import not in env.global_scope().context.future_directives):
                 self.level = -1
             else:
                 self.level = 0
         module_name = self.module_name.analyse_types(env)
         self.module_name = module_name.coerce_to_pyobject(env)
+        assert self.module_name.is_string_literal
         if self.name_list:
             name_list = self.name_list.analyse_types(env)
             self.name_list = name_list.coerce_to_pyobject(env)
-        self.is_temp = 1
+        elif '.' in self.module_name.value:
+            self.module_names = TupleNode(self.module_name.pos, args=[
+                IdentifierStringNode(self.module_name.pos, value=part, constant_result=part)
+                for part in map(StringEncoding.EncodedString, self.module_name.value.split('.'))
+            ]).analyse_types(env)
         return self
 
     gil_message = "Python import"
 
     def generate_result_code(self, code):
-        if self.name_list:
-            name_list_code = self.name_list.py_result()
+        assert self.module_name.is_string_literal
+        module_name = self.module_name.value
+
+        if self.level == 0 and not self.name_list and not self.get_top_level_module:
+            if self.module_names:
+                assert self.module_names.is_literal  # make sure we create the tuple only once
+            code.globalstate.use_utility_code(UtilityCode.load_cached("ImportDottedModule", "ImportExport.c"))
+            import_code = "__Pyx_ImportDottedModule(%s, %s)" % (
+                self.module_name.py_result(),
+                self.module_names.py_result() if self.module_names else 'NULL',
+            )
         else:
-            name_list_code = "0"
-
-        code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c"))
-        import_code = "__Pyx_Import(%s, %s, %d)" % (
-            self.module_name.py_result(),
-            name_list_code,
-            self.level)
+            code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c"))
+            import_code = "__Pyx_Import(%s, %s, %d)" % (
+                self.module_name.py_result(),
+                self.name_list.py_result() if self.name_list else '0',
+                self.level)
 
-        if (self.level <= 0 and
-                self.module_name.is_string_literal and
-                self.module_name.value in utility_code_for_imports):
-            helper_func, code_name, code_file = utility_code_for_imports[self.module_name.value]
+        if self.level <= 0 and module_name in utility_code_for_imports:
+            helper_func, code_name, code_file = utility_code_for_imports[module_name]
             code.globalstate.use_utility_code(UtilityCode.load_cached(code_name, code_file))
             import_code = '%s(%s)' % (helper_func, import_code)
 
diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py
--- a/Cython/Compiler/Parsing.py
+++ b/Cython/Compiler/Parsing.py
@@ -1675,11 +1675,6 @@ def p_import_statement(s):
                 as_name=as_name,
                 is_absolute=is_absolute)
         else:
-            if as_name and "." in dotted_name:
-                name_list = ExprNodes.ListNode(pos, args=[
-                    ExprNodes.IdentifierStringNode(pos, value=s.context.intern_ustring("*"))])
-            else:
-                name_list = None
             stat = Nodes.SingleAssignmentNode(
                 pos,
                 lhs=ExprNodes.NameNode(pos, name=as_name or target_name),
@@ -1687,7 +1682,8 @@ def p_import_statement(s):
                     pos,
                     module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name),
                     level=0 if is_absolute else None,
-                    name_list=name_list))
+                    get_top_level_module='.' in dotted_name and as_name is None,
+                    name_list=None))
         stats.append(stat)
     return Nodes.StatListNode(pos, stats=stats)
 
diff --git a/runtests.py b/runtests.py
--- a/runtests.py
+++ b/runtests.py
@@ -29,6 +29,7 @@
 except (ImportError, AttributeError):
     IS_CPYTHON = True
     IS_PYPY = False
+IS_PY2 = sys.version_info[0] < 3
 
 from io import open as io_open
 try:
@@ -1124,6 +1125,8 @@ def run_distutils(self, test_directory, module, workdir, incdir,
                     extension = newext or extension
             if self.language == 'cpp':
                 extension.language = 'c++'
+            if IS_PY2:
+                workdir = str(workdir)  # work around type check in distutils that disallows unicode strings
             build_extension.extensions = [extension]
             build_extension.build_temp = workdir
             build_extension.build_lib  = workdir
@@ -1911,7 +1914,7 @@ def __init__(self, shard_num, shard_count):
         self.shard_num = shard_num
         self.shard_count = shard_count
 
-    def __call__(self, testname, tags=None, _hash=zlib.crc32, _is_py2=sys.version_info[0] < 3):
+    def __call__(self, testname, tags=None, _hash=zlib.crc32, _is_py2=IS_PY2):
         # Cannot use simple hash() here as shard processes might use different hash seeds.
         # CRC32 is fast and simple, but might return negative values in Py2.
         hashval = _hash(testname) & 0x7fffffff if _is_py2 else _hash(testname.encode())
@@ -2493,10 +2496,7 @@ def runtests(options, cmd_args, coverage=None):
     else:
         text_runner_options = {}
         if options.failfast:
-            if sys.version_info < (2, 7):
-                sys.stderr.write("--failfast not supported with Python < 2.7\n")
-            else:
-                text_runner_options['failfast'] = True
+            text_runner_options['failfast'] = True
         test_runner = unittest.TextTestRunner(verbosity=options.verbosity, **text_runner_options)
 
     if options.pyximport_py:


Test patch:
diff --git a/tests/run/importas.pyx b/tests/run/importas.pyx
--- a/tests/run/importas.pyx
+++ b/tests/run/importas.pyx
@@ -1,4 +1,14 @@
+# mode: run
+# tag: all_language_levels
+
 __doc__ = u"""
+>>> try: sys
+... except NameError: pass
+... else: print("sys was defined!")
+>>> try: distutils
+... except NameError: pass
+... else: print("distutils was defined!")
+
 >>> import sys as sous
 >>> import distutils.core as corey
 >>> from copy import deepcopy as copey
diff --git a/tests/run/reimport_from_package.srctree b/tests/run/reimport_from_package.srctree
--- a/tests/run/reimport_from_package.srctree
+++ b/tests/run/reimport_from_package.srctree
@@ -17,8 +17,8 @@ setup(
 
 import sys
 import a
-assert a in sys.modules.values(), list(sys.modules)
-assert sys.modules['a'] is a, list(sys.modules)
+assert a in sys.modules.values(), sorted(sys.modules)
+assert sys.modules['a'] is a, sorted(sys.modules)
 
 from atest.package import module
 
@@ -33,8 +33,8 @@ assert 'atest.package.module' in sys.modules
 
 import a
 import atest.package.module as module
-assert module in sys.modules.values(), list(sys.modules)
-assert sys.modules['atest.package.module'] is module, list(sys.modules)
+assert module in sys.modules.values(), sorted(sys.modules)
+assert sys.modules['atest.package.module'] is module, sorted(sys.modules)
 
 if sys.version_info >= (3, 5):
     from . import pymodule
diff --git a/tests/run/relativeimport_T542.srctree b/tests/run/relativeimport_T542.srctree
--- a/tests/run/relativeimport_T542.srctree
+++ b/tests/run/relativeimport_T542.srctree
@@ -26,7 +26,8 @@ try:
 except ImportError:
     pass
 else:
-    assert False, "absolute import succeeded"
+    import sys
+    assert False, "absolute import succeeded: %s" % sorted(sys.modules)
 
 import relimport.a
 import relimport.bmod