Repository: pyinstaller/pyinstaller

Issue:
feat: Python 3.10 support
[Python 3.10](https://docs.python.org/3.10/) is coming. The new [`match`,`case`](https://www.python.org/dev/peps/pep-0635/) stuff is particularly exciting to a developer coming from Rust or Haskell.

PyInstaller is a great tool for the developer when she wants to distribute a software application to the end-user without requiring the end-user to have any particular version of python installed. This is can simplify the developer's maintenance efforts - rather than trying to keep her software compatible across multiple versions of Python - and can allow her to take advantage of newer language features, like those found in the newest version of Python.

It would be a great addition for PyInstaller to add support for Python 3.10.

What I have tried:
I have tried, naively, to run PyInstaller against python 3.10.0a6 but immediately ran into at least one issue:
- [`_bootlocale` has been removed in cPython 3.10](https://github.com/python/cpython/commit/b62bdf71ea0cd52041d49691d8ae3dc645bd48e1) and has been replaced with a new `_locale._get_locale_encoding()`
excluding `_bootlocale` from `PY3_BASE_MODULES` for python 3.10 in PyInstaller/compat.py allows PyInstaller to generate an executable, however from here I kept running into issues when running the resulting binary.

It is not obvious to me what needs to be done to support python 3.10, That said, if desirable I am perfectly happy to do what I can with a little direction.


Discussion:
I can get a functional executable on Linux if I:
- Replace `_bootlocale` with `_locale` as you said in `compat.PY3_BASE_MODULES`
- Add `struct` to `compat.PY3_BASE_MODULES`
- Copy/paste the importer/loader runtime scripts from `PyInstaller/loader` straight into the built application's root.

Number 3 is a hack - we'll need to work out why the original approach to inject those runtime modules isn't working. 
> Number 3 is a hack - we'll need to work out why the original approach to inject those runtime modules isn't working.

Because [`PI_PyObject_CallFunction` calls](https://github.com/pyinstaller/pyinstaller/blob/35103b3e158dcc4f062d74bf64aa7860b2d75858/bootloader/src/pyi_pythonlib.c#L561-L572) are erroring out with `PY_SSIZE_T_CLEAN macro must be defined for '#' formats`. It might be enough to have our `PI_PyObject_CallFunction` point to `_PyObject_CallFunction_SizeT` instead of `PyObject_CallFunction` for Python 3.10...
We'll need to address `pyvers` field in COOKIE becoming 40 under python 3.10... for example, encoding it as
```
pyvers = sys.version_info[0] * 100 + sys.version_info[1]
```
instead of 
```
pyvers = sys.version_info[0] * 10 + sys.version_info[1]
```
(it's a 32-bit int, so there's no shortage of space...).

Resolving patch:
diff --git a/PyInstaller/archive/writers.py b/PyInstaller/archive/writers.py
--- a/PyInstaller/archive/writers.py
+++ b/PyInstaller/archive/writers.py
@@ -443,7 +443,7 @@ def save_trailer(self, tocpos):
 
         # now save teh cookie
         total_len = tocpos + toclen + self._cookie_size
-        pyvers = sys.version_info[0] * 10 + sys.version_info[1]
+        pyvers = sys.version_info[0] * 100 + sys.version_info[1]
         # Before saving cookie we need to convert it to corresponding C representation.
         cookie = struct.pack(
             self._cookie_format, self.MAGIC, total_len, tocpos, toclen, pyvers, self._pylib_name.encode('ascii')
diff --git a/PyInstaller/compat.py b/PyInstaller/compat.py
--- a/PyInstaller/compat.py
+++ b/PyInstaller/compat.py
@@ -35,6 +35,7 @@
 is_py37 = sys.version_info >= (3, 7)
 is_py38 = sys.version_info >= (3, 8)
 is_py39 = sys.version_info >= (3, 9)
+is_py310 = sys.version_info >= (3, 10)
 
 is_win = sys.platform.startswith('win')
 is_win_10 = is_win and (platform.win32_ver()[0] == '10')
@@ -612,7 +613,6 @@ def importlib_load_source(name, pathname):
 PY3_BASE_MODULES = {
     # These modules are direct or indirect dependencies of encodings.* modules. encodings modules must be recursively
     # included to set the I/O encoding during python startup.
-    '_bootlocale',
     '_collections_abc',
     '_weakrefset',
     'abc',
@@ -640,6 +640,9 @@ def importlib_load_source(name, pathname):
     'warnings',
 }
 
+if not is_py310:
+    PY3_BASE_MODULES.add('_bootlocale')
+
 # Object types of Pure Python modules in modulegraph dependency graph.
 # Pure Python modules have code object (attribute co_code).
 PURE_PYTHON_MODULE_TYPES = {
diff --git a/PyInstaller/depend/bytecode.py b/PyInstaller/depend/bytecode.py
--- a/PyInstaller/depend/bytecode.py
+++ b/PyInstaller/depend/bytecode.py
@@ -99,9 +99,10 @@ def finditer(pattern: Pattern, string):
     ((?:(?:`EXTENDED_ARG`.)*
      `LOAD_CONST`.)*)
 
-    # Call the function. The parameter is the argument count (which may also be >256).
+    # Call the function. The parameter is the argument count (which may also be >256) if CALL_FUNCTION or CALL_METHOD
+    # are used. For CALL_FUNCTION_EX, the parameter are flags.
     ((?:`EXTENDED_ARG`.)*
-     (?:`CALL_FUNCTION`|`CALL_METHOD`).)
+     (?:`CALL_FUNCTION`|`CALL_METHOD`|`CALL_FUNCTION_EX`).)
 """
 )
 
@@ -169,7 +170,7 @@ def function_calls(code: CodeType) -> list:
     out = []
 
     for match in finditer(_call_function_bytecode, code.co_code):
-        function_root, methods, args, arg_count = match.groups()
+        function_root, methods, args, function_call = match.groups()
 
         # For foo():
         #   `function_root` contains 'foo' and `methods` is empty.
@@ -180,11 +181,22 @@ def function_calls(code: CodeType) -> list:
         function = ".".join([function_root] + methods)
 
         args = loads(args, code)
-        arg_count = extended_arguments(arg_count)
+        if function_call[0] == dis.opmap['CALL_FUNCTION_EX']:
+            flags = extended_arguments(function_call)
+            if flags != 0:
+                # Keyword arguments present. Unhandled at the moment.
+                continue
+            # In calls with const arguments, args contains a single
+            # tuple with all values.
+            if len(args) != 1 or not isinstance(args[0], tuple):
+                continue
+            args = list(args[0])
+        else:
+            arg_count = extended_arguments(function_call)
 
-        if arg_count != len(args):
-            # This happens if there are variable or keyword arguments. Bail out in either case.
-            continue
+            if arg_count != len(args):
+                # This happens if there are variable or keyword arguments. Bail out in either case.
+                continue
 
         out.append((function, args))
 


Test patch:
diff --git a/tests/functional/test_basic.py b/tests/functional/test_basic.py
--- a/tests/functional/test_basic.py
+++ b/tests/functional/test_basic.py
@@ -621,8 +621,7 @@ def test_option_runtime_tmpdir(pyi_builder):
         print('test - runtime_tmpdir - custom runtime temporary directory')
         import os
         import sys
-        if sys.platform == 'win32':
-            import win32api
+
         cwd = os.path.abspath(os.getcwd())
         runtime_tmpdir = os.path.abspath(sys._MEIPASS)
         # for onedir mode, runtime_tmpdir == cwd
diff --git a/tests/functional/test_path_encodings.py b/tests/functional/test_path_encodings.py
--- a/tests/functional/test_path_encodings.py
+++ b/tests/functional/test_path_encodings.py
@@ -15,6 +15,8 @@
 
 import pytest
 
+from PyInstaller.utils.tests import importorskip
+
 
 def test_ascii_path(pyi_builder):
     distdir = pyi_builder._distdir
@@ -121,11 +123,13 @@ def test_win_non_codepage_path(pyi_builder, monkeypatch):
 
 
 @pytest.mark.win32
+@importorskip('win32api')
 def test_win_py3_no_shortpathname(pyi_builder):
     pyi_builder.test_script('pyi_win_py3_no_shortpathname.py')
 
 
 @pytest.mark.win32
+@importorskip('win32api')
 def test_win_TEMP_has_shortpathname(pyi_builder, monkeypatch, tmp_path):
     """
     Test if script if pass if $TMP holds a short path name.
diff --git a/tests/requirements-libraries.txt b/tests/requirements-libraries.txt
--- a/tests/requirements-libraries.txt
+++ b/tests/requirements-libraries.txt
@@ -30,10 +30,10 @@ babel==2.9.1
 future==0.18.2
 gevent==21.8.0
 pygments==2.10.0
-pyside2==5.15.2
+pyside2==5.15.2; python_version < '3.10'
 pyside6==6.2.0
-pyqt5==5.15.4
-pyqtwebengine==5.15.4
+pyqt5==5.15.4; python_version < '3.10'
+pyqtwebengine==5.15.4; python_version < '3.10'
 pyqt6==6.2.0  # PyQt6 Qt6 bindings
 pyqt6-qt6==6.2.0  # Qt6 binaries (keep in sync with pyqt6 version!)
 python-dateutil==2.8.2
@@ -57,19 +57,19 @@ ipython==7.28.0; python_version > '3.6'
 ipython==7.16.1; python_version <= '3.6'  # pyup: ignore
 
 # pandas also dropped support
-pandas==1.3.3; python_version > '3.6'
+pandas==1.3.3; python_version > '3.6' and python_version < '3.10'
 pandas==1.1.5; python_version <= '3.6'  # pyup: ignore
 
 # so did numpy
-numpy==1.21.2; python_version > '3.6'
+numpy==1.21.2; python_version > '3.6' and (python_version < '3.10' or sys_platform != 'darwin')
 numpy==1.19.4; python_version <= '3.6'  # pyup: ignore
 
 # scipy too
-scipy==1.7.1; python_version > '3.6'
+scipy==1.7.1; python_version > '3.6' and python_version < '3.10'
 scipy==1.5.4; python_version <= '3.6'  # pyup: ignore
 
 # and matplotlib
-matplotlib==3.4.3; python_version > '3.6'
+matplotlib==3.4.3; python_version > '3.6' and python_version < '3.10'
 matplotlib==3.3.4; python_version <= '3.6'  # pyup: ignore
 
 # Install PyInstaller Hook Sample, to ensure that tests declared in
diff --git a/tests/requirements-tools.txt b/tests/requirements-tools.txt
--- a/tests/requirements-tools.txt
+++ b/tests/requirements-tools.txt
@@ -34,9 +34,10 @@ psutil
 # Check new flake8 violations on pull requests
 flake8
 
-pywin32; sys_platform == 'win32'
+# These are required by some of basic tests
+pywin32; sys_platform == 'win32' and python_version < '3.10'
 
-lxml
+lxml; sys_platform != 'win32' or python_version < '3.10'  # no wheels for 3.10, but we can build from sdist on non-Windows
 
 # crypto support (`--key` option)
 tinyaes ~= 1.0
diff --git a/tests/unit/test_modulegraph/test_modulegraph.py b/tests/unit/test_modulegraph/test_modulegraph.py
--- a/tests/unit/test_modulegraph/test_modulegraph.py
+++ b/tests/unit/test_modulegraph/test_modulegraph.py
@@ -8,8 +8,8 @@
 import warnings
 from altgraph import Graph
 from PyInstaller.compat import is_win
+from PyInstaller.utils.tests import importorskip
 import textwrap
-from lxml import etree
 import pickle
 
 from importlib._bootstrap_external import SourceFileLoader, ExtensionFileLoader
@@ -979,6 +979,7 @@ def test_createReference(self):
         self.assertIsInstance(e, int)
         self.assertEqual(graph.graph.edge_data(e), 'direct')
 
+    @importorskip('lxml')
     def test_create_xref(self):
         # XXX: This test is far from optimal, it just ensures
         # that all code is exercised to catch small bugs and
@@ -999,6 +1000,7 @@ def test_create_xref(self):
 
         data = fp.getvalue()
         # Don't tolerate any HTML `parsing errors <https://lxml.de/parsing.html#parser-options>`_.
+        from lxml import etree
         parser = etree.HTMLParser(recover=False)
         tree = etree.parse(StringIO(data), parser)
         assert tree is not None