Repository: takluyver/flit
Pull request #106 (2017-05-11 16:39:05)

Make sdist
Following discussion on #74, this code can make an sdist, including an autogenerated `setup.py`. I have tested making an sdist of flit itself and installing it using pip, confirming that it works.

To make an sdist, run `flit sdist` in the project directory. There is not yet code for uploading the sdist directly, but you can use twine to do so.

* Selecting the files to go into an sdist requires that the project is in a VCS, and that that VCS is available on PATH. `git` and `hg` are the currently supported options.
* It also checks whether there are any untracked files in the repo, or files deleted since the last commit, other than a few whitelisted ones (`dist/`, `__pycache__`, `*.pyc`). It will refuse to build the sdist if files have been added or removed which the VCS doesn't know about, but staging the changes (`git add`/`git rm`) is sufficient to make it happy. This check might be relaxed in the future.
* There is no work as yet to handle git submodules or hg subrepos, and I wouldn't expect this to correctly package projects using them.
* If there is a `setup.py` adjacent to `flit.ini` and checked into the repository, the sdist will use that instead of autogenerating a `setup.py` script. In that case, it's up to the package author to ensure that their `setup.py` is correct.
* A `pyproject.toml` is not generated; my plan is to fold `flit.ini` into that file, so it will exist in the source repo (#77).


Patch:
diff --git a/.travis.yml b/.travis.yml
index ed0243a..ab728e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,5 @@
 language: python
 python:
-  - "3.6"
   - "3.5"
   - "3.4"
 install: pip install -r requirements-test.txt
diff --git a/flit/__init__.py b/flit/__init__.py
index 889bdb9..f8f5424 100644
--- a/flit/__init__.py
+++ b/flit/__init__.py
@@ -43,6 +43,10 @@ def main(argv=None):
           help="Verify the package metadata with the PyPI server"
     )
 
+    parser_sdist = subparsers.add_parser('sdist',
+         help="Build a source distribution (.tar.gz)",
+    )
+
     parser_install = subparsers.add_parser('install',
         help="Install the package",
     )
@@ -88,6 +92,12 @@ def main(argv=None):
                      repo=args.repository)
         except common.ProblemInModule as e:
             sys.exit(e.args[0])
+    elif args.subcmd == 'sdist':
+        from .sdist import SdistBuilder
+        try:
+            SdistBuilder(args.ini_file).build()
+        except common.VCSError as e:
+            sys.exit(str(e))
     elif args.subcmd == 'install':
         from .install import Installer
         try:
diff --git a/flit/common.py b/flit/common.py
index b783f72..050f8eb 100644
--- a/flit/common.py
+++ b/flit/common.py
@@ -57,6 +57,15 @@ class NoDocstringError(ProblemInModule): pass
 class NoVersionError(ProblemInModule): pass
 class InvalidVersion(ProblemInModule): pass
 
+class VCSError(Exception):
+    def __init__(self, msg, directory):
+        self.msg = msg
+        self.directory = directory
+
+    def __str__(self):
+        return self.msg + ' ({})'.format(self.directory)
+
+
 @contextmanager
 def _module_load_ctx():
     """Preserve some global state that modules might change at import time.
diff --git a/flit/sdist.py b/flit/sdist.py
new file mode 100644
index 0000000..6381e4f
--- /dev/null
+++ b/flit/sdist.py
@@ -0,0 +1,251 @@
+from configparser import ConfigParser
+from collections import defaultdict
+import io
+import logging
+import os
+from pathlib import Path
+from posixpath import join as pjoin
+from pprint import pformat
+import re
+import sys
+import tarfile
+
+from flit import common, inifile
+from flit.common import VCSError
+from flit.vcs import identify_vcs
+from flit.wheel import EntryPointsConflict
+
+log = logging.getLogger(__name__)
+
+SETUP = """\
+#!/usr/bin/env python
+# setup.py generated by flit for tools that don't yet use PEP 517
+
+from distutils.core import setup
+
+{before}
+setup(name={name!r},
+      version={version!r},
+      description={description!r},
+      author={author!r},
+      author_email={author_email!r},
+      url={url!r},
+      {extra}
+     )
+"""
+
+PKG_INFO = """\
+Metadata-Version: 1.1
+Name: {name}
+Version: {version}
+Summary: {summary}
+Home-page: {home_page}
+Author: {author}
+Author-email: {author_email}
+"""
+
+
+def auto_packages(pkgdir: str):
+    """Discover subpackages and package_data"""
+    pkgdir = os.path.normpath(pkgdir)
+    pkg_name = os.path.basename(pkgdir)
+    pkg_data = defaultdict(list)
+    # Undocumented distutils feature: the empty string matches all package names
+    pkg_data[''].append('*')
+    packages = [pkg_name]
+    subpkg_paths = set()
+
+    def find_nearest_pkg(rel_path):
+        parts = rel_path.split(os.sep)
+        for i in reversed(range(1, len(parts))):
+            ancestor = os.sep.join(parts[:i])
+            if ancestor in subpkg_paths:
+                pkg = '.'.join([pkg_name] + parts[:i])
+                return pkg, os.sep.join(parts[i:])
+
+        # Relative to the top-level package
+        return pkg_name, rel_path
+
+    for path, dirnames, filenames in os.walk(pkgdir, topdown=True):
+        if os.path.basename(path) == '__pycache__':
+            continue
+
+        from_top_level = os.path.relpath(path, pkgdir)
+        if from_top_level == '.':
+            continue
+
+        is_subpkg = '__init__.py' in filenames
+        if is_subpkg:
+            subpkg_paths.add(from_top_level)
+            parts = from_top_level.split(os.sep)
+            packages.append('.'.join([pkg_name] + parts))
+        else:
+            pkg, from_nearest_pkg = find_nearest_pkg(from_top_level)
+            pkg_data[pkg].append(os.path.join(from_nearest_pkg, '*'))
+
+    return packages, dict(pkg_data)
+
+def _parse_req(requires_dist):
+    """Parse "Foo (v); python_version == '2.x'" from Requires-Dist
+
+    Returns pip-style appropriate for requirements.txt.
+    """
+    if ';' in requires_dist:
+        name_version, env_mark = requires_dist.split(';', 1)
+        env_mark = env_mark.strip()
+    else:
+        name_version, env_mark = requires_dist, None
+
+    if '(' in name_version:
+        # turn 'name (X)' and 'name (<X.Y)'
+        # into 'name == X' and 'name < X.Y'
+        name, version = name_version.split('(', 1)
+        name = name.strip()
+        version = version.replace(')', '').strip()
+        if not any(c in version for c in '=<>'):
+            version = '==' + version
+        name_version = name + version
+
+    return name_version, env_mark
+
+def convert_requires(metadata):
+    install_reqs = []
+    extra_reqs = defaultdict(list)
+    for req in metadata.requires_dist:
+        name_version, env_mark = _parse_req(req)
+        if env_mark is None:
+            install_reqs.append(name_version)
+        else:
+            extra_reqs[':'+env_mark].append(name_version)
+
+    return install_reqs, dict(extra_reqs)
+
+def include_path(p):
+    return not (p.startswith('dist/')
+                or ('/__pycache__' in p)
+                or p.endswith('.pyc'))
+
+class SdistBuilder:
+    def __init__(self, ini_path=Path('flit.ini')):
+        self.ini_path = ini_path
+        self.ini_info = inifile.read_pkg_ini(ini_path)
+        self.module = common.Module(self.ini_info['module'], ini_path.parent)
+        self.metadata = common.make_metadata(self.module, self.ini_info)
+        self.srcdir = ini_path.parent
+
+    def prep_entry_points(self):
+        # Write entry points
+        cp = ConfigParser()
+
+        if self.ini_info['scripts']:
+            cp['console_scripts'] = {k: '%s:%s' % v
+                                 for (k, v) in self.ini_info['scripts'].items()}
+
+        if self.ini_info['entry_points_file'] is not None:
+            cp.read(str(self.ini_info['entry_points_file']))
+            if 'console_scripts' in cp:
+                raise EntryPointsConflict
+
+        res = defaultdict(list)
+        for group in cp.sections():
+            sect = cp[group]
+            for name in sorted(sect):
+                res[group].append('{} = {}'.format(name, sect[name]))
+
+        return dict(res)
+
+    def find_tracked_files(self):
+        vcs_mod = identify_vcs(self.srcdir)
+        untracked_deleted = vcs_mod.list_untracked_deleted_files(self.srcdir)
+        if list(filter(include_path, untracked_deleted)):
+            raise VCSError("Untracked or deleted files in the source directory. "
+                           "Commit, undo or ignore these files in your VCS.",
+                           self.srcdir)
+
+        files = vcs_mod.list_tracked_files(self.srcdir)
+        return sorted(filter(include_path, files))
+
+    def make_setup_py(self):
+        before, extra = [], []
+        if self.module.is_package:
+            packages, package_data = auto_packages(str(self.module.path))
+            before.append("packages = \\\n%s\n" % pformat(sorted(packages)))
+            before.append("package_data = \\\n%s\n" % pformat(package_data))
+            extra.append("packages=packages,")
+            extra.append("package_data=package_data,")
+        else:
+            extra.append("py_modules={!r},".format([self.module.name]))
+
+        install_reqs, extra_reqs = convert_requires(self.metadata)
+        if install_reqs:
+            before.append("install_requires = \\\n%s\n" % pformat(install_reqs))
+            extra.append("install_requires=install_requires,")
+        if extra_reqs:
+            before.append("extras_require = \\\n%s\n" % pformat(extra_reqs))
+            extra.append("extras_require=extras_require,")
+
+        entrypoints = self.prep_entry_points()
+        if entrypoints:
+            before.append("entry_points = \\\n%s\n" % pformat(entrypoints))
+            extra.append("entry_points=entry_points,")
+
+        if self.metadata.requires_python:
+            extra.append('python_requires=%r,' % self.metadata.requires_python)
+
+        return SETUP.format(
+            before='\n'.join(before),
+            name=self.metadata.name,
+            version=self.metadata.version,
+            description=self.metadata.summary,
+            author=self.metadata.author,
+            author_email=self.metadata.author_email,
+            url=self.metadata.home_page,
+            extra='\n      '.join(extra),
+        ).encode('utf-8')
+
+    def build(self, target_dir:Path =None):
+        if target_dir is None:
+            target_dir = self.ini_path.parent / 'dist'
+        if not target_dir.exists():
+            target_dir.mkdir(parents=True)
+        target = target_dir / '{}-{}.tar.gz'.format(
+                        self.metadata.name, self.metadata.version)
+        tf = tarfile.open(str(target), mode='w:gz')
+        tf_dir = '{}-{}'.format(self.metadata.name, self.metadata.version)
+
+        files_to_add = self.find_tracked_files()
+
+        for relpath in files_to_add:
+            path = self.srcdir / relpath
+            tf.add(str(path), arcname=pjoin(tf_dir, relpath))
+
+        if 'setup.py' in files_to_add:
+            log.warning("Using setup.py from repository, not generating setup.py")
+        else:
+            setup_py = self.make_setup_py()
+            log.info("Writing generated setup.py")
+            ti = tarfile.TarInfo(pjoin(tf_dir, 'setup.py'))
+            ti.size = len(setup_py)
+            tf.addfile(ti, io.BytesIO(setup_py))
+
+        pkg_info = PKG_INFO.format(
+            name=self.metadata.name,
+            version=self.metadata.version,
+            summary=self.metadata.summary,
+            home_page=self.metadata.home_page,
+            author=self.metadata.author,
+            author_email=self.metadata.author_email,
+        ).encode('utf-8')
+        ti = tarfile.TarInfo(pjoin(tf_dir, 'PKG-INFO'))
+        ti.size = len(pkg_info)
+        tf.addfile(ti, io.BytesIO(pkg_info))
+
+        tf.close()
+
+        log.info("Built %s", target)
+
+if __name__ == '__main__':
+    try:
+        SdistBuilder().build()
+    except VCSError as e:
+        sys.exit(str(e))
diff --git a/flit/vcs/__init__.py b/flit/vcs/__init__.py
new file mode 100644
index 0000000..12f3b87
--- /dev/null
+++ b/flit/vcs/__init__.py
@@ -0,0 +1,14 @@
+from pathlib import Path
+
+from flit.common import VCSError
+from . import hg
+from . import git
+
+def identify_vcs(directory: Path):
+    for p in [directory] + list(directory.parents):
+        if (p / '.git').is_dir():
+            return git
+        if (p / '.hg').is_dir():
+            return hg
+
+    raise VCSError("Directory does not appear to be in a VCS", directory)
diff --git a/flit/vcs/git.py b/flit/vcs/git.py
new file mode 100644
index 0000000..d4697d0
--- /dev/null
+++ b/flit/vcs/git.py
@@ -0,0 +1,12 @@
+import os
+from subprocess import check_output
+
+def list_tracked_files(directory):
+    outb = check_output(['git', 'ls-files'], cwd=str(directory))
+    return [os.fsdecode(l) for l in outb.strip().splitlines()]
+
+def list_untracked_deleted_files(directory):
+    outb = check_output(['git', 'ls-files', '--deleted', '--others',
+                         '--exclude-standard'],
+                        cwd=str(directory))
+    return [os.fsdecode(l) for l in outb.strip().splitlines()]
diff --git a/flit/vcs/hg.py b/flit/vcs/hg.py
new file mode 100644
index 0000000..e3817b3
--- /dev/null
+++ b/flit/vcs/hg.py
@@ -0,0 +1,31 @@
+import os
+from subprocess import check_output
+
+def find_repo_root(directory):
+    for p in [directory] + list(directory.parents):
+        if (p / '.hg').is_dir():
+            return p
+
+def _repo_paths_to_directory_paths(paths, directory):
+    # 'hg status' gives paths from repo root, which may not be our directory.
+    repo = find_repo_root(directory)
+    if directory != repo:
+        directory_in_repo = str(directory.relative_to(repo)) + os.sep
+        ix = len(directory_in_repo)
+        paths = [p[ix:] for p in paths
+                 if os.path.normpath(p).startswith(directory_in_repo)]
+    return paths
+
+
+def list_tracked_files(directory):
+    outb = check_output(['hg', 'status', '--clean', '--added', '--no-status'],
+                        cwd=str(directory))
+    paths = [os.fsdecode(l) for l in outb.strip().splitlines()]
+    return _repo_paths_to_directory_paths(paths, directory)
+
+
+def list_untracked_deleted_files(directory):
+    outb = check_output(['hg', 'status', '--unknown', '--deleted', '--no-status'],
+                        cwd=str(directory))
+    paths = [os.fsdecode(l) for l in outb.strip().splitlines()]
+    return _repo_paths_to_directory_paths(paths, directory)
diff --git a/flit/vendorized/__init__.py b/flit/vendorized/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/flit/vendorized/readme/__init__.py b/flit/vendorized/readme/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/flit/wheel.py b/flit/wheel.py
index 28e4a23..ae7b3d9 100644
--- a/flit/wheel.py
+++ b/flit/wheel.py
@@ -1,15 +1,14 @@
 from base64 import urlsafe_b64encode
 import configparser
 import contextlib
+import tempfile
 from datetime import datetime
-import glob
 import hashlib
 import io
 import logging
 import os
 import re
 import sys
-import tempfile
 
 if sys.version_info >= (3, 6):
     import zipfile
@@ -66,8 +65,8 @@ class WheelBuilder:
     def wheel_filename(self):
         tag = ('py2.' if self.supports_py2 else '') + 'py3-none-any'
         return '{}-{}-{}.whl'.format(
-                re.sub("[^\w\d.]+", "_", self.metadata.name, flags=re.UNICODE),
-                re.sub("[^\w\d.]+", "_", self.metadata.version, flags=re.UNICODE),
+                re.sub("[^\w\d.]+", "_", self.metadata.name, re.UNICODE),
+                re.sub("[^\w\d.]+", "_", self.metadata.version, re.UNICODE),
                 tag)
 
     def _include(self, path):
@@ -156,9 +155,6 @@ class WheelBuilder:
         elif self.ini_info['entry_points_file'] is not None:
             self._add_file(self.ini_info['entry_points_file'],
                            dist_info + '/entry_points.txt')
-        for base in ('COPYING', 'LICENSE'):
-            for path in sorted(self.directory.glob(base + '*')):
-                self._add_file(path, '%s/%s' % (dist_info, path.name))
 
         with self._write_to_zip(dist_info + '/WHEEL') as f:
             f.write(wheel_file_template)


Test patch:
diff --git a/tests/samples/package1/data_dir/foo.sh b/tests/samples/package1/data_dir/foo.sh
new file mode 100644
index 0000000..92abcfb
--- /dev/null
+++ b/tests/samples/package1/data_dir/foo.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "Example data file"
diff --git a/tests/samples/package1/subpkg/__init__.py b/tests/samples/package1/subpkg/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/samples/package1/subpkg/sp_data_dir/test.json b/tests/samples/package1/subpkg/sp_data_dir/test.json
new file mode 100644
index 0000000..f77d03c
--- /dev/null
+++ b/tests/samples/package1/subpkg/sp_data_dir/test.json
@@ -0,0 +1,1 @@
+{"example": true}
diff --git a/tests/samples/package1/subpkg2/__init__.py b/tests/samples/package1/subpkg2/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_sdist.py b/tests/test_sdist.py
new file mode 100644
index 0000000..fd6cdfb
--- /dev/null
+++ b/tests/test_sdist.py
@@ -0,0 +1,88 @@
+import ast
+from os.path import join as pjoin
+from pathlib import Path
+import pytest
+from shutil import which, copy
+import sys
+from tempfile import TemporaryDirectory
+from testpath import assert_isfile, MockCommand
+
+from flit import sdist
+
+samples_dir = Path(__file__).parent / 'samples'
+
+def test_auto_packages():
+    packages, pkg_data = sdist.auto_packages(str(samples_dir / 'package1'))
+    assert packages == ['package1', 'package1.subpkg', 'package1.subpkg2']
+    assert pkg_data == {'': ['*'],
+                        'package1': ['data_dir/*'],
+                        'package1.subpkg': ['sp_data_dir/*'],
+                       }
+
+def test_make_sdist():
+    # Smoke test of making a complete sdist
+    if not which('git'):
+        pytest.skip("requires git")
+    builder = sdist.SdistBuilder(samples_dir / 'package1-pkg.ini')
+    with TemporaryDirectory() as td:
+        td = Path(td)
+        builder.build(td)
+        assert_isfile(td / 'package1-0.1.tar.gz')
+
+
+LIST_FILES = """\
+#!{python}
+import sys
+from os.path import join
+if '--deleted' not in sys.argv:
+    print('foo')
+    print(join('dir1', 'bar'))
+    print(join('dir1', 'subdir', 'qux'))
+    print(join('dir2', 'abc'))
+    print(join('dist', 'def'))
+""".format(python=sys.executable)
+
+
+def test_get_files_list_git():
+    with TemporaryDirectory() as td:
+        copy(str(samples_dir / 'module1.py'), td)
+        copy(str(samples_dir / 'module1-pkg.ini'), td)
+        td = Path(td)
+        (td / '.git').mkdir()
+        builder = sdist.SdistBuilder(td / 'module1-pkg.ini')
+        with MockCommand('git', LIST_FILES):
+            files = builder.find_tracked_files()
+
+        assert set(files) == {
+            'foo', pjoin('dir1', 'bar'), pjoin('dir1', 'subdir', 'qux'),
+            pjoin('dir2', 'abc')
+        }
+
+def test_get_files_list_hg():
+    with TemporaryDirectory() as td:
+        dir1 = Path(td, 'dir1')
+        dir1.mkdir()
+        copy(str(samples_dir / 'module1.py'), str(dir1))
+        copy(str(samples_dir / 'module1-pkg.ini'), str(dir1))
+        td = Path(td)
+        (td / '.hg').mkdir()
+        builder = sdist.SdistBuilder(dir1 / 'module1-pkg.ini')
+        with MockCommand('hg', LIST_FILES):
+            files = builder.find_tracked_files()
+
+        assert set(files) == {
+            'bar', pjoin('subdir', 'qux')
+        }
+
+def test_make_setup_py():
+    builder = sdist.SdistBuilder(samples_dir / 'package1-pkg.ini')
+    setup = builder.make_setup_py()
+    setup_ast = ast.parse(setup)
+    # Select only assignment statements
+    setup_ast.body = [n for n in setup_ast.body if isinstance(n, ast.Assign)]
+    ns = {}
+    exec(compile(setup_ast, filename="setup.py", mode="exec"), ns)
+    assert ns['packages'] == ['package1', 'package1.subpkg', 'package1.subpkg2']
+    assert 'install_requires' not in ns
+    assert ns['entry_points'] == \
+           {'console_scripts': ['pkg_script = package1:main']}