Repository: pyinstaller/pyinstaller

Issue:
ctypes dll scan - absolute paths not working
This is a followup from #1609. PyInstaller can not handle absolute paths. The absolute path will be there in the frozen app, too, but the dll will not. We are thinking about checking this in scan_code_for_ctypes and fail is an absolute path occurs as the first. element.

How to reproduce this issue: Use `vlc.py` from https://wiki.videolan.org/PythonBinding/ and freeze it. Example output on linux:

```
$ python3 pyinstaller.py --clean vlctest/vlc.py 
102 INFO: PyInstaller: 3.1.dev0+8ae75f0
102 INFO: Python: 3.5.0
103 INFO: Platform: Linux-4.2.3-1-ARCH-x86_64-with-arch
103 INFO: wrote /home/tgc/projects/OpenLP/pyinstaller/vlc/vlc.spec
104 INFO: UPX is not available.
105 INFO: Removing temporary files and cleaning cache in /home/tgc/.local/share/pyinstaller
106 INFO: Extending PYTHONPATH with /home/tgc/projects/OpenLP/pyinstaller/vlctest
106 INFO: checking Analysis
106 INFO: Building Analysis because out00-Analysis.toc is non existent
106 INFO: Initializing module dependency graph...
107 INFO: Initializing module graph hooks...
108 INFO: Analyzing base_library.zip ...
2809 INFO: running Analysis out00-Analysis.toc
2834 INFO: Analyzing /home/tgc/projects/OpenLP/pyinstaller/vlctest/vlc.py
3041 INFO: Looking for import hooks ...
3043 INFO: Processing hook   hook-pydoc.py
3044 INFO: Processing hook   hook-encodings.py
3054 INFO: Processing hook   hook-xml.py
3275 INFO: Processing hook   hook-xml.sax.py
3285 INFO: Looking for ctypes DLLs
3357 WARNING: library libvlc.dll required via ctypes not found
3388 WARNING: library libvlc.dll required via ctypes not found
3420 WARNING: library libvlc.dylib required via ctypes not found
Traceback (most recent call last):
  File "pyinstaller.py", line 15, in <module>
    run()
  File "/home/tgc/projects/OpenLP/pyinstaller/PyInstaller/__main__.py", line 97, in run
    run_build(opts, spec_file, pyi_config)
  File "/home/tgc/projects/OpenLP/pyinstaller/PyInstaller/__main__.py", line 46, in run_build
    PyInstaller.building.build_main.main(pyi_config, spec_file, **opts.__dict__)
  File "/home/tgc/projects/OpenLP/pyinstaller/PyInstaller/building/build_main.py", line 738, in main
    build(specfile, kw.get('distpath'), kw.get('workpath'), kw.get('clean_build'))
  File "/home/tgc/projects/OpenLP/pyinstaller/PyInstaller/building/build_main.py", line 681, in build
    exec(text, spec_namespace)
  File "<string>", line 16, in <module>
  File "/home/tgc/projects/OpenLP/pyinstaller/PyInstaller/building/build_main.py", line 208, in __init__
    self.__postinit__()
  File "/home/tgc/projects/OpenLP/pyinstaller/PyInstaller/building/datastruct.py", line 156, in __postinit__
    self.assemble()
  File "/home/tgc/projects/OpenLP/pyinstaller/PyInstaller/building/build_main.py", line 469, in assemble
    ctypes_binaries, ctypes_warnings = scan_code_for_ctypes(co)
  File "/home/tgc/projects/OpenLP/pyinstaller/PyInstaller/depend/utils.py", line 181, in scan_code_for_ctypes
    elif binary != os.path.basename(binary):
  File "/usr/lib/python3.5/posixpath.py", line 139, in basename
    i = p.rfind(sep) + 1
AttributeError: 'tuple' object has no attribute 'rfind'
```



Discussion:

Resolving patch:
diff --git a/PyInstaller/building/build_main.py b/PyInstaller/building/build_main.py
--- a/PyInstaller/building/build_main.py
+++ b/PyInstaller/building/build_main.py
@@ -466,9 +466,8 @@ def assemble(self):
         for name, co in ctypes_code_objs.items():
             # Get dlls that might be needed by ctypes.
             logger.debug('Scanning %s for shared libraries or dlls', name)
-            ctypes_binaries, ctypes_warnings = scan_code_for_ctypes(co)
+            ctypes_binaries = scan_code_for_ctypes(co)
             self.binaries.extend(set(ctypes_binaries))
-            # TODO: print and write ctypes_warnings
 
         # Analyze run-time hooks.
         # Run-time hooks has to be executed before user scripts. Add them
diff --git a/PyInstaller/depend/analysis.py b/PyInstaller/depend/analysis.py
--- a/PyInstaller/depend/analysis.py
+++ b/PyInstaller/depend/analysis.py
@@ -167,7 +167,13 @@ def _safe_import_module(self, module_basename, module_name, parent_package):
         `pre_safe_import_module()`), that hook will be run _before_ the
         superclass method is called.
 
-        See superclass method for parameter and return value descriptions.
+        Pre-Safe-Import-Hooks are performed just *prior* to importing
+        the module. When running the hook, the modules parent package
+        has already been imported and ti's `__path__` is set up. But
+        the module is just about to be imported.
+
+        See the superclass method for description of parameters and
+        return value.
         """
         # If this module has pre-safe import module hooks, run these first.
         if module_name in self._hooks_pre_safe_import_module:
diff --git a/PyInstaller/depend/utils.py b/PyInstaller/depend/utils.py
--- a/PyInstaller/depend/utils.py
+++ b/PyInstaller/depend/utils.py
@@ -147,28 +147,15 @@ def pass1(code):
 
 
 def scan_code_for_ctypes(co):
-    instrs = pass1(co.co_code)
-    warnings = []
     binaries = []
 
-    for i in range(len(instrs)):
-        # ctypes scanning requires a scope wider than one bytecode
-        # instruction, so the code resides in a separate function
-        # for clarity.
-        ctypesb, ctypesw = scan_code_instruction_for_ctypes(co, instrs, i)
-        binaries.extend(ctypesb)
-        warnings.extend(ctypesw)
-
-    for c in co.co_consts:
-        if isinstance(c, type(co)):
-            nested_binaries, nested_warnings = scan_code_for_ctypes(c)
-            binaries.extend(nested_binaries)
-            warnings.extend(nested_warnings)
+    __recursivly_scan_code_objects_for_ctypes(co, binaries)
 
     # If any of the libraries has been requested with anything
     # different then the bare filename, drop that entry and warn
     # the user - pyinstaller would need to patch the compiled pyc
     # file to make it work correctly!
+    binaries = set(binaries)
     for binary in list(binaries):
         # 'binary' might be in some cases None. Some Python
         # modules might contain code like the following. For
@@ -180,13 +167,32 @@ def scan_code_for_ctypes(co):
             binaries.remove(binary)
         elif binary != os.path.basename(binary):
             # TODO make these warnings show up somewhere.
-            warnings.append("W: ignoring %s - ctypes imports only supported using bare filenames" % binary)
+            loggger.warn("ignoring %s - ctypes imports only supported using bare filenames" % binary)
 
     binaries = _resolveCtypesImports(binaries)
-    return binaries, warnings
+    return binaries
+
+
+def __recursivly_scan_code_objects_for_ctypes(co, binaries):
+    # Note: `binaries` is a list, which gets extended here.
+    instructions = iter(pass1(co.co_code))
+    while 1:
+        # ctypes scanning requires a scope wider than one bytecode
+        # instruction, so the code resides in a separate function
+        # for clarity.
+        try:
+            bin = __scan_code_instruction_for_ctypes(co, instructions)
+            if bin:
+                binaries.append(bin)
+        except StopIteration:
+            break
+
+    for c in co.co_consts:
+        if isinstance(c, type(co)):
+            __recursivly_scan_code_objects_for_ctypes(c, binaries)
 
 
-def scan_code_instruction_for_ctypes(co, instrs, i):
+def __scan_code_instruction_for_ctypes(co, instructions):
     """
     Detects ctypes dependencies, using reasonable heuristics that
     should cover most common ctypes usages; returns a tuple of two
@@ -194,23 +200,21 @@ def scan_code_instruction_for_ctypes(co, instrs, i):
     dependencies, the other containing warnings.
     """
 
-    def _libFromConst(i):
+    def _libFromConst():
         """Extracts library name from an expected LOAD_CONST instruction and
         appends it to local binaries list.
         """
-        op, oparg, conditional, curline = instrs[i]
+        op, oparg, conditional, curline = next(instructions)
         if op == LOAD_CONST:
             soname = co.co_consts[oparg]
-            binaries.add(soname)
+            return soname
 
-    warnings = []
-    binaries = set()
 
-    op, oparg, conditional, curline = instrs[i]
+    op, oparg, conditional, curline = next(instructions)
     expected_ops = (LOAD_GLOBAL, LOAD_NAME)
 
     if op not in expected_ops:
-        return [], []
+        return None
 
     name = co.co_names[oparg]
     if name == "ctypes":
@@ -223,11 +227,10 @@ def _libFromConst(i):
         #
         # In this case "strip" the `ctypes` by advancing and expecting
         # `LOAD_ATTR` next.
-        i += 1
         expected_ops = (LOAD_ATTR,)
-        op, oparg, conditional, curline = instrs[i]
+        op, oparg, conditional, curline = next(instructions)
         if op not in expected_ops:
-            return [], []
+            return None
         name = co.co_names[oparg]
 
     if name in ("CDLL", "WinDLL", "OleDLL", "PyDLL"):
@@ -235,7 +238,7 @@ def _libFromConst(i):
         #
         #   LOAD_GLOBAL 0 (CDLL) <--- we "are" here right now
         #   LOAD_CONST 1 ('library.so')
-        _libFromConst(i + 1)
+        return _libFromConst()
 
     elif name in ("cdll", "windll", "oledll", "pydll"):
         # Guesses ctypes imports of these types:
@@ -250,36 +253,32 @@ def _libFromConst(i):
         #     LOAD_GLOBAL   0 (cdll) <--- we "are" here right now
         #     LOAD_ATTR     1 (LoadLibrary)
         #     LOAD_CONST    1 ('library.so')
-        i += 1
-        op, oparg, conditional, curline = instrs[i]
+        op, oparg, conditional, curline = next(instructions)
         if op == LOAD_ATTR:
             if co.co_names[oparg] == "LoadLibrary":
                 # Second type, needs to fetch one more instruction
-                _libFromConst(i + 1)
+                return _libFromConst()
             else:
                 # First type
-                soname = co.co_names[oparg] + ".dll"
-                binaries.add(soname)
+                return co.co_names[oparg] + ".dll"
+
     elif op == LOAD_ATTR and name in ("util", ):
         # Guesses ctypes imports of these types::
         #
         #  ctypes.util.find_library('gs')
         #
-        #     LOAD_GLOBAL   0 (ctype)
+        #     LOAD_GLOBAL   0 (ctypes)
         #     LOAD_ATTR     1 (util) <--- we "are" here right now
         #     LOAD_ATTR     1 (find_library)
         #     LOAD_CONST    1 ('gs')
-        i += 1
-        op, oparg, conditional, curline = instrs[i]
+        op, oparg, conditional, curline = next(instructions)
         if op == LOAD_ATTR:
             if co.co_names[oparg] == "find_library":
-                i += 1
-                op, oparg, conditional, curline = instrs[i]
+                op, oparg, conditional, curline = next(instructions)
                 if op == LOAD_CONST:
                     libname = co.co_consts[oparg]
-                    soname = ctypes.util.find_library(libname)
-                    binaries.add(soname)
-    return binaries, warnings
+                    return ctypes.util.find_library(libname)
+
 
 
 # TODO Reuse this code with modulegraph implementation


Test patch:
diff --git a/tests/functional/test_import.py b/tests/functional/test_import.py
--- a/tests/functional/test_import.py
+++ b/tests/functional/test_import.py
@@ -127,17 +127,6 @@ def skip_if_lib_missing(libname, text=None):
         print('>>> file found')
     """
 
-_template_ctypes_test = """
-        print(lib)
-        assert lib is not None and lib._name is not None
-        import sys, os
-        if getattr(sys, 'frozen', False):
-            libfile = os.path.join(sys._MEIPASS, %(soname)r)
-            print(libfile)
-            assert os.path.isfile(libfile), '%(soname)s is missing'
-            print('>>> file found')
-    """
-
 # Ghostscript's libgs.so should be available in may Unix/Linux systems
 #
 # At least on Linux, we can not use our own `ctypes_dylib` because
@@ -153,6 +142,17 @@ def test_ctypes_CDLL_find_library__gs(pyi_builder):
 
 #-- Generate test-cases for the different types of ctypes objects.
 
+_template_ctypes_test = """
+        print(lib)
+        assert lib is not None and lib._name is not None
+        import sys, os
+        if getattr(sys, 'frozen', False):
+            libfile = os.path.join(sys._MEIPASS, %(soname)r)
+            print(libfile)
+            assert os.path.isfile(libfile), '%(soname)s is missing'
+            print('>>> file found')
+    """
+
 parameters = []
 ids = []
 for prefix in ('', 'ctypes.'):
@@ -184,12 +184,39 @@ def test_ctypes_gen(pyi_builder, monkeypatch, funcname, compiled_dylib, test_id)
         import ctypes ; from ctypes import *
         lib = %s(%%(soname)r)
     """ % funcname + _template_ctypes_test
-    source = source +_template_ctypes_test
 
     __monkeypatch_resolveCtypesImports(monkeypatch, compiled_dylib.dirname)
     pyi_builder.test_source(source % locals(), test_id=test_id)
 
 
+@pytest.mark.parametrize("funcname,test_id", parameters, ids=ids)
+def test_ctypes_in_func_gen(pyi_builder, monkeypatch, funcname,
+                            compiled_dylib, test_id):
+    """
+    This is much like test_ctypes_gen except that the ctypes calls
+    are in a function. See issue #1620.
+    """
+    # Workaround, see above.
+    # :todo: remove this workaround (see above)
+    if not is_win and funcname.endswith(("WinDLL", "OleDLL")):
+        pytest.skip('%s requires windows' % funcname)
+
+    soname = compiled_dylib.basename
+
+    source = ("""
+    import ctypes ; from ctypes import *
+    def f():
+      def g():
+        lib = %s(%%(soname)r)
+    """ % funcname +
+    _template_ctypes_test + """
+      g()
+    f()
+    """)
+    __monkeypatch_resolveCtypesImports(monkeypatch, compiled_dylib.dirname)
+    pyi_builder.test_source(source % locals(), test_id=test_id)
+
+
 # TODO: Add test-cases for the prefabricated library loaders supporting
 # attribute accesses on windows. Example::
 #