Repository: pyinstaller/pyinstaller

Issue:
Executable cannot detect nested folder structure
I am trying to import chromadb's module to my executable. However, the executable is not correctly handling a file that is nested in  another subfolder in the module. The error is coming from the importlib_resources import files. The exact path that chromadb uses is "chromadb.migrations.embeddings_queue" which seems to hold a migration file that is a sqlite3 file. I have successfully copied that file to my data set via datas += collect_data_files('chromadb.migrations.embeddings_queue') in my .spec file, however this seems to be an issue with pyinstaller not being able to handle a nested folder structure. importlib_resources would return MultiplexedPath('/opt/homebrew/lib/python3.11/site-packages/chromadb/migrations/embeddings_queue') given the function file("chromadb.migrations.embeddings_queue") however the executable created by pyinstaller returns invalid path error:

Here is the exact error:
Traceback (most recent call last):
  File "main.py", line 234, in <module>
  File "main.py", line 65, in main
  File "main.py", line 86, in ingest_documents
  File "chromadb/__init__.py", line 106, in PersistentClient
  File "chromadb/__init__.py", line 143, in Client
  File "chromadb/config.py", line 243, in instance
  File "chromadb/api/segment.py", line 81, in __init__
  File "chromadb/config.py", line 184, in require
  File "chromadb/config.py", line 243, in instance
  File "chromadb/db/impl/sqlite.py", line 65, in __init__
  File "importlib_resources/_common.py", line 46, in wrapper
  File "importlib_resources/_common.py", line 56, in files
  File "importlib_resources/_common.py", line 113, in from_package
  File "importlib_resources/_compat.py", line 85, in get_resource_reader
  File "importlib_resources/_compat.py", line 63, in _native_reader
  File "importlib_resources/_compat.py", line 60, in _available_reader
  File "<frozen importlib._bootstrap_external>", line 1380, in get_resource_reader
  File "importlib/resources/readers.py", line 110, in __init__
ValueError: Invalid path



Discussion:
This looks like some sort of incompatibility between our frozen importer, namespace packages, and importlib resources reader...

As a work-around, can you try forcing collection in pure-source mode? This should bypass our frozen importer for the package, and let python's built-in stuff handle it. I.e., in the spec file, add:
```
a = Analysis(
    ...
    module_collection_mode={        
        'chromadb': 'py',
    }
)
```
and rebuilt the application. You should see the modules of `chromadb` collected as .py files. 

You will still need to deal with hidden imports yourself, particularly for nested namespace packages (but suppose if you came this far, you already did some of that).
This worked! I needed to manually add __init__ files to these locations and add these hidden_modules:
hiddenimports = ['chromadb.telemetry.posthog', 'chromadb.api.segment', 'chromadb.db.impl', 'chromadb.db.impl.sqlite', 'chromadb.migrations', 'chromadb.migrations.embeddings_queue', 'chromadb.segment.impl.manager', 'chromadb.segment.impl.manager.local', 'chromadb.segment.impl.metadata', 'chromadb.segment.impl.metadata.sqlite', 'chromadb.segment.impl.vector', 'chromadb.segment.impl.vector.batch', 'chromadb.segment.impl.vector.brute_force_index', 'chromadb.segment.impl.vector.hnsw_params', 'chromadb.segment.impl.vector.local_hnsw', 'chromadb.segment.impl.vector.local_persistent_hnsw']

But for now it seems to be working? Its not fully running yet but there might be unrelated issues


Resolving patch:
diff --git a/PyInstaller/loader/pyimod02_importers.py b/PyInstaller/loader/pyimod02_importers.py
--- a/PyInstaller/loader/pyimod02_importers.py
+++ b/PyInstaller/loader/pyimod02_importers.py
@@ -272,11 +272,20 @@ def find_spec(self, fullname, path=None, target=None):
             return None
 
         if self._is_pep420_namespace_package(entry_name):
+            from importlib._bootstrap_external import _NamespacePath
             # PEP-420 namespace package; as per PEP 451, we need to return a spec with "loader" set to None
             # (a.k.a. not set)
             spec = _frozen_importlib.ModuleSpec(fullname, None, is_package=True)
             # Set submodule_search_locations, which seems to fill the __path__ attribute.
-            spec.submodule_search_locations = [os.path.dirname(self.get_filename(entry_name))]
+            # This needs to be an instance of `importlib._bootstrap_external._NamespacePath` for `importlib.resources`
+            # to work correctly with the namespace package; otherwise `importlib.resources.files()` throws an
+            # `ValueError('Invalid path')` due to this check:
+            # https://github.com/python/cpython/blob/v3.11.5/Lib/importlib/resources/readers.py#L109-L110
+            spec.submodule_search_locations = _NamespacePath(
+                entry_name,
+                [os.path.dirname(self.get_filename(entry_name))],
+                self,
+            )
             return spec
 
         # origin has to be the filename


Test patch:
diff --git a/tests/functional/modules/pyi_namespace_package_with_data/hooks/hook-pyi_test_nspkg.py b/tests/functional/modules/pyi_namespace_package_with_data/hooks/hook-pyi_test_nspkg.py
new file mode 100644
--- /dev/null
+++ b/tests/functional/modules/pyi_namespace_package_with_data/hooks/hook-pyi_test_nspkg.py
@@ -0,0 +1,4 @@
+from PyInstaller.utils.hooks import collect_submodules, collect_data_files
+
+hiddenimports = collect_submodules('pyi_test_nspkg')
+datas = collect_data_files('pyi_test_nspkg')
diff --git a/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/a.py b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/a.py
new file mode 100644
--- /dev/null
+++ b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/a.py
@@ -0,0 +1 @@
+#
diff --git a/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/b.py b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/b.py
new file mode 100644
--- /dev/null
+++ b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/b.py
@@ -0,0 +1 @@
+#
diff --git a/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/data/data_file1.txt b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/data/data_file1.txt
new file mode 100644
--- /dev/null
+++ b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/data/data_file1.txt
@@ -0,0 +1 @@
+Data file #1.
diff --git a/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/data/data_file2.txt b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/data/data_file2.txt
new file mode 100644
--- /dev/null
+++ b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/data/data_file2.txt
@@ -0,0 +1 @@
+Data file #2.
diff --git a/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/data/data_file3.txt b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/data/data_file3.txt
new file mode 100644
--- /dev/null
+++ b/tests/functional/modules/pyi_namespace_package_with_data/package/pyi_test_nspkg/data/data_file3.txt
@@ -0,0 +1 @@
+Data file #3.
diff --git a/tests/functional/test_importlib_resources.py b/tests/functional/test_importlib_resources.py
--- a/tests/functional/test_importlib_resources.py
+++ b/tests/functional/test_importlib_resources.py
@@ -98,3 +98,47 @@ def test_importlib_resources_frozen(pyi_builder, package_type, tmpdir, script_di
         test_script,
         pyi_args=['--paths', pathex, '--hidden-import', 'pyi_pkgres_testpkg', '--additional-hooks-dir', hooks_dir]
     )
+
+
+# A separate test for verifying that `importlib.resources.files()` works with PEP-420 namespace packages. See #7921.
+# The sub-directory containing the data files is also a PEP-410 namespace (sub)package. However, in the context of
+# PyInstaller, there are actually two slightly different possibilities:
+#  - if we collect only the data files, and the namespace package itself is not collected into PYZ, the situation is
+#    the same as in unfrozen python - python's built-in import machinery takes care of the namespace package and the
+#    associated resource reader.
+#  - if the namespace package is collected into PYZ (in addition to resources being collected as data files), the
+#    namespace package ends up being handled by PyInstaller's `PyiFrozenImporter`, which requires extra care to ensure
+#    compatibility with `importlib` resource reader.
+# The test covers both scenarios via `as_package` parameter.
+@skipif(
+    not is_py39 and not is_module_satisfies('importlib_resources'),
+    reason="Python prior to 3.9 requires importlib_resources."
+)
+@pytest.mark.parametrize('as_package', [True, False])
+def test_importlib_resources_namespace_package_data_files(pyi_builder, as_package):
+    pathex = os.path.join(_MODULES_DIR, 'pyi_namespace_package_with_data', 'package')
+    hooks_dir = os.path.join(_MODULES_DIR, 'pyi_namespace_package_with_data', 'hooks')
+    if as_package:
+        hidden_imports = ['--hidden-import', 'pyi_test_nspkg', '--hidden-import', 'pyi_test_nspkg.data']
+    else:
+        hidden_imports = ['--hidden-import', 'pyi_test_nspkg']
+    pyi_builder.test_source(
+        """
+        try:
+            import importlib_resources
+        except ModuleNotFoundError:
+            import importlib.resources as importlib_resources
+
+        # Get the package's directory (= our data directory)
+        data_dir = importlib_resources.files("pyi_test_nspkg.data")
+
+        # Sanity check; verify the directory's base name
+        assert data_dir.name == "data"
+
+        # Check that data files exist
+        assert (data_dir / "data_file1.txt").is_file()
+        assert (data_dir / "data_file2.txt").is_file()
+        assert (data_dir / "data_file3.txt").is_file()
+        """,
+        pyi_args=['--paths', pathex, *hidden_imports, '--additional-hooks-dir', hooks_dir]
+    )