Repository: pyodide/pyodide

Issue:
Update to Python 3.7
Python 3.7 is out.


Discussion:
What I have to do on this Issue. This is first time I will work on issue.
The basic steps would be to update the `Makefile` in `cpython/Makefile` to download and build Python 3.7 instead of 3.6.  If we're lucky, the patches in `cpython/patches` will still apply cleanly against Python 3.7.  If not, they will need to be updated, and any new compiler errors will need to be resolved with new patches.  Experience with building C libraries from source would be helpful here.
It would quiet bigger task for beginner. 

Resolving patch:
diff --git a/tools/buildpkg.py b/tools/buildpkg.py
--- a/tools/buildpkg.py
+++ b/tools/buildpkg.py
@@ -79,25 +79,6 @@ def patch(path, srcpath, pkg, args):
         fd.write(b'\n')
 
 
-def get_libdir(srcpath, args):
-    # Get the name of the build/lib.XXX directory that distutils wrote its
-    # output to
-    slug = subprocess.check_output([
-        str(Path(args.host) / 'bin' / 'python3'),
-        '-c',
-        'import sysconfig, sys; '
-        'print("{}-{}.{}".format('
-        'sysconfig.get_platform(), '
-        'sys.version_info[0], '
-        'sys.version_info[1]))']).decode('ascii').strip()
-    purelib = srcpath / 'build' / 'lib'
-    if purelib.is_dir():
-        libdir = purelib
-    else:
-        libdir = srcpath / 'build' / ('lib.' + slug)
-    return libdir
-
-
 def compile(path, srcpath, pkg, args):
     if (srcpath / '.built').is_file():
         return
@@ -121,10 +102,11 @@ def compile(path, srcpath, pkg, args):
 
     post = pkg.get('build', {}).get('post')
     if post is not None:
-        libdir = get_libdir(srcpath, args)
+        site_packages_dir = (
+            srcpath / 'install' / 'lib' / 'python3.7' / 'site-packages')
         pkgdir = path.parent.resolve()
         env = {
-            'BUILD': libdir,
+            'SITEPACKAGES': site_packages_dir,
             'PKGDIR': pkgdir
         }
         subprocess.run([
diff --git a/tools/common.py b/tools/common.py
--- a/tools/common.py
+++ b/tools/common.py
@@ -2,8 +2,8 @@
 
 
 ROOTDIR = Path(__file__).parent.resolve()
-HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.6.4' / 'host'
-TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.6.4'
+HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.7.0' / 'host'
+TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.7.0'
 DEFAULTCFLAGS = ''
 DEFAULTLDFLAGS = ' '.join([
     '-O3',


Test patch:
diff --git a/cpython/patches/testing.patch b/cpython/patches/testing.patch
--- a/cpython/patches/testing.patch
+++ b/cpython/patches/testing.patch
@@ -1,5 +1,5 @@
 diff --git a/Lib/platform.py b/Lib/platform.py
-index cc2db9870d..ac4e3c538f 100755
+index 20f9817f4f..bcf96874c0 100755
 --- a/Lib/platform.py
 +++ b/Lib/platform.py
 @@ -748,7 +748,7 @@ def _syscmd_uname(option, default=''):
@@ -21,18 +21,18 @@ index cc2db9870d..ac4e3c538f 100755
          return default
      target = _follow_symlinks(target)
 diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py
-index ca5f9c20dd..97934039ee 100644
+index 64b25aab80..7cf67fd149 100644
 --- a/Lib/test/support/script_helper.py
 +++ b/Lib/test/support/script_helper.py
-@@ -11,6 +11,7 @@ import subprocess
+@@ -9,6 +9,7 @@ import os.path
+ import subprocess
  import py_compile
- import contextlib
- import shutil
-+import unittest
  import zipfile
++import unittest
  
  from importlib.util import source_from_cache
-@@ -37,6 +38,8 @@ def interpreter_requires_environment():
+ from test.support import make_legacy_pyc, strip_python_stderr
+@@ -34,6 +35,8 @@ def interpreter_requires_environment():
      situation.  PYTHONPATH or PYTHONUSERSITE are other common environment
      variables that might impact whether or not the interpreter can start.
      """
@@ -40,16 +40,16 @@ index ca5f9c20dd..97934039ee 100644
 +
      global __cached_interp_requires_environment
      if __cached_interp_requires_environment is None:
-         # Try running an interpreter with -E to see if it works or not.
-@@ -165,6 +168,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
+         # If PYTHONHOME is set, assume that we need it
+@@ -172,6 +175,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
      kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
      object.
      """
 +    raise unittest.SkipTest("no subprocess")
 +
-     cmd_line = [sys.executable, '-E']
-     cmd_line.extend(args)
-     # Under Fedora (?), GNU readline can output junk on stderr when initialized,
+     cmd_line = [sys.executable]
+     if not interpreter_requires_environment():
+         cmd_line.append('-E')
 diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
 index 55faf4c427..b2201c09e7 100644
 --- a/Lib/test/test_code.py
@@ -67,7 +67,7 @@ index 55faf4c427..b2201c09e7 100644
  import weakref
  try:
 diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
-index b73a96f757..0d1c411f9a 100644
+index 1fc4de11e1..e6c91707ae 100644
 --- a/Lib/test/test_import/__init__.py
 +++ b/Lib/test/test_import/__init__.py
 @@ -10,7 +10,10 @@ import py_compile
diff --git a/test/make_test_list.py b/test/make_test_list.py
--- a/test/make_test_list.py
+++ b/test/make_test_list.py
@@ -7,7 +7,7 @@
 
 
 TEST_DIR = (Path(__file__).parent
-            / "cpython/build/3.6.4/host/lib/python3.6/test")
+            / "cpython/build/3.6.4/host/lib/python3.7/test")
 
 
 def collect_tests(base_dir):
diff --git a/test/python_tests.txt b/test/python_tests.txt
--- a/test/python_tests.txt
+++ b/test/python_tests.txt
@@ -4,8 +4,10 @@
 # Following reason codes are skipped:
 # - platform-specific: This is testing something about a particular platform
 #   that isn't relevant here
+# - async: relies on async
 # - audioop: Requires the audioop module
 # - floating point: Failures caused by floating-point differences
+# - threading: Failures due to lack of a threading implementation
 # - subprocess: Failures caused by no subprocess module. Some of these are
 #   because the underlying functionality depends on subprocess, and others are
 #   just a side-effect of the way the test is written. The latter should
@@ -17,7 +19,8 @@
 # - strftime: Failures due to differences / shortcomings in WebAssembly's
 #   implementation of date/time formatting in strftime and strptime
 # - permissions: Issues with the test writing to the virtual filesystem
-# - locale: Fails due to include locale implementation.
+# - locale: Fails due to limitations in the included locale implementation.
+# - multiprocessing: Fails due to no multiprocessing implementation.
 # - nonsense: This functionality doesn't make sense in this context. Includes
 #   things like `pip`, `distutils`
 #
@@ -28,7 +31,7 @@
 # - crash-chrome: Same as crash but only affecting Chrome
 # - crash-firefox: Same as crash but only affecting Firefox
 
-test___all__
+test___all__       multiprocessing
 test___future__
 test__locale  locale
 test__opcode
@@ -40,30 +43,31 @@ test_argparse
 test_array
 test_asdl_parser
 test_ast
-test_asyncgen
-test_asynchat
-test_asyncio.test_base_events
-test_asyncio.test_events
-test_asyncio.test_futures
-test_asyncio.test_locks
-test_asyncio.test_pep492
-test_asyncio.test_proactor_events
-test_asyncio.test_queues
-test_asyncio.test_selector_events
-test_asyncio.test_sslproto
-test_asyncio.test_streams
-test_asyncio.test_subprocess
-test_asyncio.test_tasks
-test_asyncio.test_transports
-test_asyncio.test_unix_events
-test_asyncio.test_windows_events
-test_asyncio.test_windows_utils
-test_asyncore bad ioctl syscall
+test_asyncgen async
+test_asynchat async
+test_asyncio.test_base_events async
+test_asyncio.test_events      async
+test_asyncio.test_futures     async
+test_asyncio.test_locks       async
+test_asyncio.test_pep492      async
+test_asyncio.test_proactor_events async
+test_asyncio.test_queues          async
+test_asyncio.test_selector_events async
+test_asyncio.test_sslproto        async
+test_asyncio.test_streams         async
+test_asyncio.test_subprocess      async
+test_asyncio.test_tasks           async
+test_asyncio.test_transports      async
+test_asyncio.test_unix_events     async
+test_asyncio.test_windows_events  async
+test_asyncio.test_windows_utils   async
+test_asyncore bad ioctl syscall   async
 test_atexit
 test_audioop  audioop
 test_augassign
 test_base64
 test_baseexception
+test_bdb
 test_bigaddrspace
 test_bigmem
 test_binascii
@@ -76,6 +80,7 @@ test_bufio
 test_builtin  floating point
 test_bytes
 test_bz2
+test_c_locale_coercion
 test_calendar
 test_call
 test_capi
@@ -112,16 +117,19 @@ test_complex
 test_concurrent_futures
 test_configparser
 test_contains
+test_context
 test_contextlib
+test_contextlib_async async
 test_copy
 test_copyreg  dbm
-test_coroutines
+test_coroutines async
 test_cprofile _lsprof
 test_crashers
 test_crypt
 test_csv
 test_ctypes
 test_curses
+test_dataclasses
 test_datetime strftime
 test_dbm      permissions
 test_dbm_dumb permissions
@@ -143,7 +151,7 @@ test_dis
 test_distutils crash
 test_doctest   subprocess
 test_doctest2
-test_docxmlrpc
+test_docxmlrpc  socket
 test_dtrace platform
 test_dummy_thread
 test_dummy_threading
@@ -155,7 +163,7 @@ test_email.test__header_value_parser
 test_email.test_asian_codecs
 test_email.test_contentmanager
 test_email.test_defect_handling
-test_email.test_email
+test_email.test_email threading
 test_email.test_generator
 test_email.test_headerregistry
 test_email.test_inversion
@@ -164,8 +172,9 @@ test_email.test_parser
 test_email.test_pickleable
 test_email.test_policy
 test_email.test_utils
+test_embed
 test_ensurepip  nonsense
-test_enum
+test_enum       threading
 test_enumerate
 test_eof
 test_epoll  crash
@@ -185,14 +194,15 @@ test_finalization
 test_float  floating point
 test_flufl
 test_fnmatch
-test_fork1
+test_fork1  threading
 test_format
 test_fractions
 test_frame
+test_frozen
 test_fstring
-test_ftplib
+test_ftplib syscall 21537
 test_funcattrs
-test_functools
+test_functools  threading
 test_future
 test_future3
 test_future4
@@ -201,6 +211,7 @@ test_gc
 test_gdb
 test_generator_stop
 test_generators
+test_genericclass
 test_genericpath  permissions
 test_genexps
 test_getargs2
@@ -213,7 +224,7 @@ test_grammar
 test_grp
 test_gzip
 test_hash
-test_hashlib
+test_hashlib  threading
 test_heapq
 test_hmac
 test_html
@@ -221,9 +232,9 @@ test_htmlparser
 test_http_cookiejar
 test_http_cookies
 test_httplib  socket
-test_httpservers
+test_httpservers  threading
 test_idle
-test_imaplib
+test_imaplib  socket
 test_imghdr
 test_imp  crash
 test_importlib.builtin.test_finder
@@ -244,15 +255,19 @@ test_importlib.import_.test_packages
 test_importlib.import_.test_path
 test_importlib.import_.test_relative_imports
 test_importlib.source.test_case_sensitivity
-test_importlib.source.test_file_loader
+test_importlib.source.test_file_loader  unittest has no attribute mock
 test_importlib.source.test_finder
 test_importlib.source.test_path_hook
 test_importlib.source.test_source_encoding
 test_importlib.test_abc
 test_importlib.test_api
 test_importlib.test_lazy
-test_importlib.test_locks
+test_importlib.test_locks threading
 test_importlib.test_namespace_pkgs
+test_importlib.test_open
+test_importlib.test_path
+test_importlib.test_read
+test_importlib.test_resource
 test_importlib.test_spec
 test_importlib.test_util
 test_importlib.test_windows platform-specific
@@ -298,7 +313,6 @@ test_long
 test_longexp
 test_lzma
 test_macpath  platform-specific
-test_macurl2path
 test_mailbox  crash
 test_mailcap  nonsense
 test_marshal
@@ -338,14 +352,14 @@ test_peepholer
 test_pickle dbm
 test_pickletools  dbm
 test_pipes        platform-specific
-test_pkg
+test_pkg          unknown
 test_pkgimport
 test_pkgutil
 test_platform   subprocess
 test_plistlib
 test_poll subprocess
 test_popen  subprocess
-test_poplib
+test_poplib bad ioctl syscall 21537
 test_posix  crash
 test_posixpath  crash
 test_pow
@@ -361,10 +375,10 @@ test_py_compile
 test_pyclbr
 test_pydoc  crash
 test_pyexpat
-test_queue
+test_queue  threading
 test_quopri subprocess
 test_raise
-test_random
+test_random subprocess
 test_range
 test_re locale
 test_readline
@@ -374,10 +388,10 @@ test_reprlib
 test_resource
 test_richcmp
 test_rlcompleter  crash
-test_robotparser
+test_robotparser  socket
 test_runpy
 test_sax
-test_sched
+test_sched  threading
 test_scope
 test_script_helper
 test_secrets
@@ -392,7 +406,7 @@ test_signal
 test_site subprocess
 test_slice
 test_smtpd
-test_smtplib
+test_smtplib  baad ioctl syscall 21537
 test_smtpnet
 test_sndhdr audioop
 test_socket
@@ -425,20 +439,20 @@ test_symtable
 test_syntax
 test_sys  subprocess
 test_sys_setprofile
-test_sys_settrace
+test_sys_settrace async
 test_sysconfig  nonsense
 test_syslog
 test_tarfile  crash
 test_tcl
-test_telnetlib
+test_telnetlib  bad ioctl syscall 21537
 test_tempfile crash
 test_textwrap
-test_thread
-test_threaded_import
-test_threadedtempfile
-test_threading
-test_threading_local
-test_threadsignals
+test_thread threading
+test_threaded_import  floating point
+test_threadedtempfile threading
+test_threading        threading
+test_threading_local  threading
+test_threadsignals    threading
 test_time
 test_timeit
 test_timeout
@@ -477,7 +491,7 @@ test_unpack
 test_unpack_ex
 test_urllib crash
 test_urllib2  subprocess
-test_urllib2_localnet
+test_urllib2_localnet socket
 test_urllib2net
 test_urllib_response
 test_urllibnet
@@ -485,14 +499,15 @@ test_urlparse
 test_userdict
 test_userlist
 test_userstring
+test_utf8_mode
 test_utf8source
 test_uu
 test_uuid subprocess
 test_venv crash
-test_wait3
-test_wait4
+test_wait3  threading
+test_wait4  threading
 test_wave crash
-test_weakref
+test_weakref  threading
 test_weakset
 test_webbrowser replaced
 test_winconsoleio
@@ -506,6 +521,7 @@ test_xml_etree
 test_xml_etree_c
 test_xmlrpc networking
 test_xmlrpc_net
+test_xxtestfuzz
 test_yield_from
 test_zipapp
 test_zipfile
diff --git a/test/test_common.py b/test/test_common.py
--- a/test/test_common.py
+++ b/test/test_common.py
@@ -43,4 +43,7 @@ def test_import(name, selenium_standalone):
 
     for import_name in meta.get('test', {}).get('imports', []):
         selenium_standalone.load_package(name)
-        selenium_standalone.run('import %s' % import_name)
+        try:
+            selenium_standalone.run('import %s' % import_name)
+        except Exception as e:
+            print(selenium_standalone.logs)
diff --git a/test/test_pandas.py b/test/test_pandas.py
--- a/test/test_pandas.py
+++ b/test/test_pandas.py
@@ -6,7 +6,7 @@ def test_pandas(selenium, request):
         request.applymarker(pytest.mark.xfail(
             run=False, reason='chrome not supported'))
     selenium.load_package("pandas")
-    assert len(selenium.run("import pandas\ndir(pandas)")) == 179
+    assert len(selenium.run("import pandas\ndir(pandas)")) == 140
 
 
 def test_extra_import(selenium, request):
diff --git a/test/test_python.py b/test/test_python.py
--- a/test/test_python.py
+++ b/test/test_python.py
@@ -297,9 +297,15 @@ def test_run_core_python_test(python_test, selenium, request):
     try:
         selenium.run(
             "from test.libregrtest import main\n"
-            "main(['{}'], verbose=True, verbose3=True)".format(name))
+            "try:\n"
+            "    main(['{}'], verbose=True, verbose3=True)\n"
+            "except SystemExit as e:\n"
+            "    if e.code != 0:\n"
+            "        raise RuntimeError(f'Failed with code: {{e.code}}')\n"
+            .format(name))
     except selenium.JavascriptException as e:
-        assert 'SystemExit: 0' in str(e)
+        print(selenium.logs)
+        raise
 
 
 def pytest_generate_tests(metafunc):
@@ -326,18 +332,6 @@ def pytest_generate_tests(metafunc):
                              ids=test_modules_ids)
 
 
-def test_recursive_repr(selenium):
-    assert not selenium.run(
-        "d = {}\n"
-        "d[42] = d.values()\n"
-        "result = True\n"
-        "try:\n"
-        "   repr(d)\n"
-        "except RecursionError:\n"
-        "   result = False\n"
-        "result")
-
-
 def test_load_package_after_convert_string(selenium):
     """
     See #93.