Repository: pyodide/pyodide

Issue:
It's 2018, let's use `pathlib`
`pathlib` is cool.  Should update the tools to use it.


Discussion:

Resolving patch:
diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -1,11 +1,12 @@
 import json
-import os
+from pathlib import Path
 import re
 import subprocess
 import sys
 
-sys.path.insert(0, os.path.abspath(
-     os.path.join(os.path.dirname(__file__), '..', 'test')))
+sys.path.insert(
+    0, str((Path(__file__).resolve().parents[1] / 'test')))
+
 import conftest
 
 
@@ -14,11 +15,12 @@
 
 def run_native(hostpython, code):
     output = subprocess.check_output(
-        [os.path.abspath(hostpython), '-c', code],
-        cwd=os.path.dirname(__file__),
+        [hostpython.resolve(), '-c', code],
+        cwd=Path(__file__).resolve().parent,
         env={
             'PYTHONPATH':
-            os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src'))}
+            str(Path(__file__).resolve().parents[1] / 'src')
+        }
     )
     return float(output.strip().split()[-1])
 
@@ -72,12 +74,12 @@ def parse_numpy_benchmark(filename):
 
 
 def get_numpy_benchmarks():
-    root = '../numpy-benchmarks/benchmarks'
-    for filename in os.listdir(root):
-        name = os.path.splitext(filename)[0]
+    root = Path('../numpy-benchmarks/benchmarks')
+    for filename in root.iterdir():
+        name = filename.name
         if name in SKIP:
             continue
-        content = parse_numpy_benchmark(os.path.join(root, filename))
+        content = parse_numpy_benchmark(filename)
         content += (
             "import numpy as np\n"
             "_ = np.empty(())\n"
@@ -103,6 +105,6 @@ def main(hostpython):
 
 
 if __name__ == '__main__':
-    results = main(sys.argv[-2])
+    results = main(Path(sys.argv[-2]).resolve())
     with open(sys.argv[-1], 'w') as fp:
         json.dump(results, fp)
diff --git a/tools/buildpkg.py b/tools/buildpkg.py
--- a/tools/buildpkg.py
+++ b/tools/buildpkg.py
@@ -7,6 +7,7 @@
 import argparse
 import hashlib
 import os
+from pathlib import Path
 import shutil
 import subprocess
 
@@ -14,7 +15,7 @@
 import common
 
 
-ROOTDIR = os.path.abspath(os.path.dirname(__file__))
+ROOTDIR = Path(__file__).parent.resolve()
 
 
 def check_checksum(path, pkg):
@@ -37,40 +38,39 @@ def check_checksum(path, pkg):
 
 
 def download_and_extract(buildpath, packagedir, pkg, args):
-    tarballpath = os.path.join(
-        buildpath, os.path.basename(pkg['source']['url']))
-    if not os.path.isfile(tarballpath):
+    tarballpath = buildpath / Path(pkg['source']['url']).name
+    if not tarballpath.is_file():
         subprocess.run([
-            'wget', '-q', '-O', tarballpath, pkg['source']['url']
+            'wget', '-q', '-O', str(tarballpath), pkg['source']['url']
         ], check=True)
         check_checksum(tarballpath, pkg)
-    srcpath = os.path.join(buildpath, packagedir)
-    if not os.path.isdir(srcpath):
-        shutil.unpack_archive(tarballpath, buildpath)
+    srcpath = buildpath / packagedir
+    if not srcpath.is_dir():
+        shutil.unpack_archive(str(tarballpath), str(buildpath))
     return srcpath
 
 
 def patch(path, srcpath, pkg, args):
-    if os.path.isfile(os.path.join(srcpath, '.patched')):
+    if (srcpath / '.patched').is_file():
         return
 
     # Apply all of the patches
-    orig_dir = os.getcwd()
-    pkgdir = os.path.abspath(os.path.dirname(path))
+    orig_dir = Path.cwd()
+    pkgdir = path.parent.resolve()
     os.chdir(srcpath)
     try:
         for patch in pkg['source'].get('patches', []):
             subprocess.run([
-                'patch', '-p1', '--binary', '-i', os.path.join(pkgdir, patch)
+                'patch', '-p1', '--binary', '-i', pkgdir / patch
             ], check=True)
     finally:
         os.chdir(orig_dir)
 
     # Add any extra files
     for src, dst in pkg['source'].get('extras', []):
-        shutil.copyfile(os.path.join(pkgdir, src), os.path.join(srcpath, dst))
+        shutil.copyfile(pkgdir / src, srcpath / dst)
 
-    with open(os.path.join(srcpath, '.patched'), 'wb') as fd:
+    with open(srcpath / '.patched', 'wb') as fd:
         fd.write(b'\n')
 
 
@@ -78,31 +78,31 @@ def get_libdir(srcpath, args):
     # Get the name of the build/lib.XXX directory that distutils wrote its
     # output to
     slug = subprocess.check_output([
-        os.path.join(args.host, 'bin', 'python3'),
+        str(Path(args.host) / 'bin' / 'python3'),
         '-c',
         'import sysconfig, sys; '
         'print("{}-{}.{}".format('
         'sysconfig.get_platform(), '
         'sys.version_info[0], '
         'sys.version_info[1]))']).decode('ascii').strip()
-    purelib = os.path.join(srcpath, 'build', 'lib')
-    if os.path.isdir(purelib):
+    purelib = srcpath / 'build' / 'lib'
+    if purelib.is_dir():
         libdir = purelib
     else:
-        libdir = os.path.join(srcpath, 'build', 'lib.' + slug)
+        libdir = srcpath / 'build' / ('lib.' + slug)
     return libdir
 
 
 def compile(path, srcpath, pkg, args):
-    if os.path.isfile(os.path.join(srcpath, '.built')):
+    if (srcpath / '.built').is_file():
         return
 
-    orig_dir = os.getcwd()
+    orig_dir = Path.cwd()
     os.chdir(srcpath)
     try:
         subprocess.run([
-            os.path.join(args.host, 'bin', 'python3'),
-            os.path.join(ROOTDIR, 'pywasmcross'),
+            str(Path(args.host) / 'bin' / 'python3'),
+            str(ROOTDIR / 'pywasmcross'),
             '--cflags',
             args.cflags + ' ' +
             pkg.get('build', {}).get('cflags', ''),
@@ -117,7 +117,7 @@ def compile(path, srcpath, pkg, args):
     post = pkg.get('build', {}).get('post')
     if post is not None:
         libdir = get_libdir(srcpath, args)
-        pkgdir = os.path.abspath(os.path.dirname(path))
+        pkgdir = path.parent.resolve()
         env = {
             'BUILD': libdir,
             'PKGDIR': pkgdir
@@ -125,46 +125,46 @@ def compile(path, srcpath, pkg, args):
         subprocess.run([
             'bash', '-c', post], env=env, check=True)
 
-    with open(os.path.join(srcpath, '.built'), 'wb') as fd:
+    with open(srcpath / '.built', 'wb') as fd:
         fd.write(b'\n')
 
 
 def package_files(buildpath, srcpath, pkg, args):
-    if os.path.isfile(os.path.join(buildpath, '.packaged')):
+    if (buildpath / '.pacakaged').is_file():
         return
 
     name = pkg['package']['name']
     libdir = get_libdir(srcpath, args)
     subprocess.run([
         'python2',
-        os.path.join(os.environ['EMSCRIPTEN'], 'tools', 'file_packager.py'),
-        os.path.join(buildpath, name + '.data'),
+        Path(os.environ['EMSCRIPTEN']) / 'tools' / 'file_packager.py',
+        buildpath / (name + '.data'),
         '--preload',
         '{}@/lib/python3.6/site-packages'.format(libdir),
-        '--js-output={}'.format(os.path.join(buildpath, name + '.js')),
+        '--js-output={}'.format(buildpath / (name + '.js')),
         '--export-name=pyodide',
         '--exclude', '*.wasm.pre',
         '--exclude', '__pycache__',
         '--use-preload-plugins'], check=True)
     subprocess.run([
         'uglifyjs',
-        os.path.join(buildpath, name + '.js'),
+        buildpath / (name + '.js'),
         '-o',
-        os.path.join(buildpath, name + '.js')], check=True)
+        buildpath / (name + '.js')], check=True)
 
-    with open(os.path.join(buildpath, '.packaged'), 'wb') as fd:
+    with open(buildpath / '.packaged', 'wb') as fd:
         fd.write(b'\n')
 
 
 def build_package(path, args):
     pkg = common.parse_package(path)
     packagedir = pkg['package']['name'] + '-' + pkg['package']['version']
-    dirpath = os.path.dirname(path)
-    orig_path = os.getcwd()
+    dirpath = path.parent
+    orig_path = Path.cwd()
     os.chdir(dirpath)
     try:
-        buildpath = os.path.join(dirpath, 'build')
-        if not os.path.exists(buildpath):
+        buildpath = dirpath / 'build'
+        if not buildpath.is_dir():
             os.makedirs(buildpath)
         srcpath = download_and_extract(buildpath, packagedir, pkg, args)
         patch(path, srcpath, pkg, args)
@@ -195,7 +195,7 @@ def parse_args():
 
 
 def main(args):
-    path = os.path.abspath(args.package[0])
+    path = Path(args.package[0]).resolve()
     build_package(path, args)
 
 
diff --git a/tools/common.py b/tools/common.py
--- a/tools/common.py
+++ b/tools/common.py
@@ -1,11 +1,9 @@
-import os
+from pathlib import Path
 
 
-ROOTDIR = os.path.abspath(os.path.dirname(__file__))
-HOSTPYTHON = os.path.abspath(
-    os.path.join(ROOTDIR, '..', 'cpython', 'build', '3.6.4', 'host'))
-TARGETPYTHON = os.path.abspath(
-    os.path.join(ROOTDIR, '..', 'cpython', 'installs', 'python-3.6.4'))
+ROOTDIR = Path(__file__).parent.resolve()
+HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.6.4' / 'host'
+TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.6.4'
 DEFAULTCFLAGS = ''
 DEFAULTLDFLAGS = ' '.join([
     '-O3',


Test patch:
diff --git a/test/make_test_list.py b/test/make_test_list.py
--- a/test/make_test_list.py
+++ b/test/make_test_list.py
@@ -3,21 +3,24 @@
 """
 
 import os
+from pathlib import Path
 
 tests = []
 
-TEST_DIR = "../cpython/build/3.6.4/host/lib/python3.6/test"
+TEST_DIR = Path("../cpython/build/3.6.4/host/lib/python3.6/test")
+
 for root, dirs, files in os.walk(
         "../cpython/build/3.6.4/host/lib/python3.6/test"):
-    root = os.path.relpath(root, TEST_DIR)
+    root = Path(root).relative_to(TEST_DIR)
     if root == '.':
         root = ''
     else:
         root = '.'.join(root.split('/')) + '.'
 
     for filename in files:
-        if filename.startswith("test_") and filename.endswith(".py"):
-            tests.append(root + os.path.splitext(filename)[0])
+        filename = Path(filename)
+        if str(filename).startswith("test_") and filename.suffix == ".py":
+            tests.append(str(root / filename.stem))
 
 tests.sort()
 with open("python_tests.txt", "w") as fp:
diff --git a/test/test_python.py b/test/test_python.py
--- a/test/test_python.py
+++ b/test/test_python.py
@@ -1,5 +1,5 @@
 import os
-import pathlib
+from pathlib import Path
 import time
 
 
@@ -295,8 +295,7 @@ def pytest_generate_tests(metafunc):
         test_modules = []
         if 'CIRCLECI' not in os.environ or True:
             with open(
-                    str(pathlib.Path(__file__).parents[0] /
-                        "python_tests.txt")) as fp:
+                    Path(__file__).parent / "python_tests.txt") as fp:
                 for line in fp:
                     line = line.strip()
                     if line.startswith('#'):