Repository: NoRedInk/make-lambda-package

Issue:
Building for python 3.6
This project looks awesome. We are currently using Zappa, but aren't satisfied with it's bloat or packaging behavior.

One big thing missing for us is the ability to package a python 3.6 lambda application.

I believe it would be a pretty simple addition, but want to make sure it would be a welcome PR, and that my approach would be acceptable.

The only two changes I had to make locally were:

https://github.com/NoRedInk/make-lambda-package/blob/master/make_lambda_package/deps.py#L58

and 

https://github.com/NoRedInk/make-lambda-package/blob/master/make_lambda_package/archive.py#L36

The rest seemed to work just fine. So I think it would be safe to put the python version behind another cli argument that defaults to 'python2.7' and also allows 'python3.6'. That argument could then be passed to `build_deps` and `make_archive`.

If a PR with that approach would be welcome, I would be happy to whip it up. Thanks!


Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
index 533d729..ee0fba9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,15 +3,15 @@ services:
   - docker
 language: python
 cache: pip
-python:
- - "3.6"
- - "3.5"
- - "3.4"
- - "2.7"
- - "pypy"
-env:
-  - TEST_PIP_VERSION="<10.0"
-  - TEST_PIP_VERSION=""
+jobs:
+  include:
+  - python: "3.7"
+    env: TEST_PIP_VERSION="<10.0"
+  - python: "3.7"
+  - python: "3.6"
+  - python: "3.5"
+  - python: "3.4"
+  - python: "pypy"
 install:
   - "pip install tox tox-travis"
 script:
diff --git a/MANIFEST.in b/MANIFEST.in
index fd7ba93..67147d7 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,3 @@
 include *.py LICENSE README.md tox.ini .travis.yml
 recursive-include tests *.py *.txt
-exclude README.in Pipfile Pipfile.lock
+exclude README.in Pipfile Pipfile.lock shell.nix
diff --git a/Pipfile b/Pipfile
index e73bec3..0ee2735 100644
--- a/Pipfile
+++ b/Pipfile
@@ -12,4 +12,4 @@ pytest = "*"
 pytest-mock = "*"
 
 [requires]
-python_version = "3.6"
+python_version = "3.7"
diff --git a/Pipfile.lock b/Pipfile.lock
index 4ae0aa5..6dfe21a 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,11 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "87a956a22fd265eeaa77d81e159b170e5a41fda8f735afb046e14c1135b1791c"
+            "sha256": "8cd530a9c49c4b7d36496b7a8c705dea9d28a75a5a8340d58aeb657727c00956"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "3.6"
+            "python_version": "3.7"
         },
         "sources": [
             {
@@ -18,10 +18,10 @@
     "default": {
         "click": {
             "hashes": [
-                "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d",
-                "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b"
+                "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
+                "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
             ],
-            "version": "==6.7"
+            "version": "==7.0"
         },
         "e1839a8": {
             "editable": true,
@@ -29,94 +29,121 @@
         },
         "six": {
             "hashes": [
-                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
-                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
             ],
-            "version": "==1.11.0"
+            "version": "==1.12.0"
         }
     },
     "develop": {
         "atomicwrites": {
             "hashes": [
-                "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585",
-                "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6"
+                "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
+                "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
             ],
-            "version": "==1.1.5"
+            "version": "==1.3.0"
         },
         "attrs": {
             "hashes": [
-                "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265",
-                "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b"
+                "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
+                "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
             ],
-            "version": "==18.1.0"
+            "version": "==19.3.0"
         },
         "click": {
             "hashes": [
-                "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d",
-                "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b"
+                "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
+                "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
             ],
-            "version": "==6.7"
+            "version": "==7.0"
         },
-        "first": {
+        "importlib-metadata": {
             "hashes": [
-                "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e",
-                "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e"
+                "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26",
+                "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"
             ],
-            "version": "==2.0.1"
+            "markers": "python_version < '3.8'",
+            "version": "==0.23"
         },
         "more-itertools": {
             "hashes": [
-                "sha256:2b6b9893337bfd9166bee6a62c2b0c9fe7735dcf85948b387ec8cba30e85d8e8",
-                "sha256:6703844a52d3588f951883005efcf555e49566a48afd4db4e965d69b883980d3",
-                "sha256:a18d870ef2ffca2b8463c0070ad17b5978056f403fb64e3f15fe62a52db21cc0"
+                "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
+                "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
             ],
-            "version": "==4.2.0"
+            "version": "==7.2.0"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
+                "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
+            ],
+            "version": "==19.2"
         },
         "pip-tools": {
             "hashes": [
-                "sha256:90bbe6731a6a34d339bf14d90cf2892475386c7d06c458208191ac9992110e0a",
-                "sha256:f11fc3bf1d87a0b4a68d4d595f619814e2396e92d75d7bdd2500edbf002ea6de"
+                "sha256:123174aabf7f4a63dd6e0bfc8aeeb5eaddbecb75a41e9f0dd4c447b1f2de14f7",
+                "sha256:5427ea4dcc175649723985fbcace9b2d8f46f9adbcc63bc2d7b247d9bcc74917"
             ],
             "index": "pypi",
-            "version": "==2.0.2"
+            "version": "==4.2.0"
         },
         "pluggy": {
             "hashes": [
-                "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff",
-                "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c",
-                "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5"
+                "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6",
+                "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"
             ],
-            "version": "==0.6.0"
+            "version": "==0.13.0"
         },
         "py": {
             "hashes": [
-                "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7",
-                "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e"
+                "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
+                "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
+            ],
+            "version": "==1.8.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80",
+                "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"
             ],
-            "version": "==1.5.4"
+            "version": "==2.4.2"
         },
         "pytest": {
             "hashes": [
-                "sha256:0453c8676c2bee6feb0434748b068d5510273a916295fd61d306c4f22fbfd752",
-                "sha256:4b208614ae6d98195430ad6bde03641c78553acee7c83cec2e85d613c0cd383d"
+                "sha256:27abc3fef618a01bebb1f0d6d303d2816a99aa87a5968ebc32fe971be91eb1e6",
+                "sha256:58cee9e09242937e136dbb3dab466116ba20d6b7828c7620f23947f37eb4dae4"
             ],
             "index": "pypi",
-            "version": "==3.6.3"
+            "version": "==5.2.2"
         },
         "pytest-mock": {
             "hashes": [
-                "sha256:53801e621223d34724926a5c98bd90e8e417ce35264365d39d6c896388dcc928",
-                "sha256:d89a8209d722b8307b5e351496830d5cc5e192336003a485443ae9adeb7dd4c0"
+                "sha256:b3514caac35fe3f05555923eabd9546abce11571cc2ddf7d8615959d04f2c89e",
+                "sha256:ea502c3891599c26243a3a847ccf0b1d20556678c528f86c98e3cd6d40c5cf11"
             ],
             "index": "pypi",
-            "version": "==1.10.0"
+            "version": "==1.11.2"
         },
         "six": {
             "hashes": [
-                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
-                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+            ],
+            "version": "==1.12.0"
+        },
+        "wcwidth": {
+            "hashes": [
+                "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
+                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
             ],
-            "version": "==1.11.0"
+            "version": "==0.1.7"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
+                "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
+            ],
+            "version": "==0.6.0"
         }
     }
 }
diff --git a/README.in b/README.in
index ca34e5e..dabdeb4 100644
--- a/README.in
+++ b/README.in
@@ -19,7 +19,7 @@ Or pip:
 Packaging dependencies with the `--requirements-file` option requires
 [the `docker` command](https://www.docker.com/) to be in your `$PATH`.
 Dependencies will be built in a [lambci/lambda:build-python2.7](https://github.com/lambci/docker-lambda)
-container.
+container by default.
 
 
 # Usage
diff --git a/README.md b/README.md
index 7ea41e3..451efda 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ Or pip:
 Packaging dependencies with the `--requirements-file` option requires
 [the `docker` command](https://www.docker.com/) to be in your `$PATH`.
 Dependencies will be built in a [lambci/lambda:build-python2.7](https://github.com/lambci/docker-lambda)
-container.
+container by default.
 
 
 # Usage
@@ -68,6 +68,9 @@ To use it:
                                       package. Useful for config files.
       --work-dir <output_directory>   Where to store intermediary files and the
                                       zipped package.
+      --runtime <lambda_runtime>      Lambda runtime. Docker image
+                                      `lambci/lambda:build-{runtime}` will be used
+                                      for the build.
       --help                          Show this message and exit.
 
 # Development
diff --git a/make_lambda_package/archive.py b/make_lambda_package/archive.py
index d7217c9..25d3bbf 100644
--- a/make_lambda_package/archive.py
+++ b/make_lambda_package/archive.py
@@ -7,6 +7,7 @@ from make_lambda_package import fsutil
 
 def make_archive(
         paths,
+        runtime,
         repo_source_files=None,
         local_source_files=None,
         deps_file=None):
@@ -17,7 +18,7 @@ def make_archive(
         if local_source_files:
             _add_local_files(f, local_source_files)
         if deps_file:
-            _add_deps(f, paths, deps_file)
+            _add_deps(f, paths, deps_file, runtime)
 
 
 def _add_local_files(zipfile, local_source_files):
@@ -31,9 +32,9 @@ def _add_repo_files(zipfile, paths, repo_source_files):
             zipfile.write(path)
 
 
-def _add_deps(zipfile, paths, deps_file):
+def _add_deps(zipfile, paths, deps_file, runtime):
     site_packages_path = os.path.join(
-        paths.build_dir, 'env', 'lib', 'python2.7', 'site-packages')
+        paths.build_dir, 'env', 'lib', runtime, 'site-packages')
     with fsutil.chdir(site_packages_path), open(deps_file) as f:
         for line in f:
             if line.startswith(os.pardir):
diff --git a/make_lambda_package/cli.py b/make_lambda_package/cli.py
index e600fdd..0216742 100644
--- a/make_lambda_package/cli.py
+++ b/make_lambda_package/cli.py
@@ -26,12 +26,18 @@ from make_lambda_package import scm
               metavar='<output_directory>',
               type=click.Path(exists=False, file_okay=False, writable=True),
               help='Where to store intermediary files and the zipped package. ')
+@click.option('--runtime',
+              metavar='<lambda_runtime>',
+              default='python2.7',
+              type=click.Choice(['python2.7', 'python3.6', 'python3.7']),
+              help='Lambda runtime. Docker image `lambci/lambda:build-{runtime}` will be used for the build.')
 def main(
         source,
         repo_source_files,
         requirements_file,
         local_source_file,
-        work_dir):
+        work_dir,
+        runtime):
     """
     Bundle up a deployment package for AWS Lambda.
 
@@ -93,11 +99,12 @@ def main(
     deps_file = None
     if requirements_file:
         click.echo('Building deps..')
-        deps_file = deps.build_deps(paths, requirements_file)
+        deps_file = deps.build_deps(paths, requirements_file, runtime)
 
     click.echo('Creating zip file..')
     archive.make_archive(
         paths,
+        runtime,
         repo_source_files,
         local_source_file,
         deps_file)
diff --git a/make_lambda_package/deps.py b/make_lambda_package/deps.py
index 9821a4d..8293854 100644
--- a/make_lambda_package/deps.py
+++ b/make_lambda_package/deps.py
@@ -34,7 +34,7 @@ done
 '''
 
 
-def build_deps(paths, requirements_file):
+def build_deps(paths, requirements_file, runtime):
     requirements = parse_requirements(
         os.path.join(paths.src_dir, requirements_file), session=True)
     package_names = [req.name for req in requirements]
@@ -55,7 +55,7 @@ def build_deps(paths, requirements_file):
         'docker', 'run',
         '-v', '{build_dir}:{docker_build_dir}'.format(**context),
         '-v', '{src_dir}:{docker_src_dir}'.format(**context),
-        'lambci/lambda:build-python2.7',
+        'lambci/lambda:build-{runtime}'.format(runtime=runtime),
         'bash', '-c', build_script,
     ]
     subprocess.check_call(command)
diff --git a/setup.py b/setup.py
index ef87539..a85acac 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,6 @@ setup(
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
         'Topic :: System :: Software Distribution',
     ]
diff --git a/shell.nix b/shell.nix
new file mode 100644
index 0000000..9994af4
--- /dev/null
+++ b/shell.nix
@@ -0,0 +1,9 @@
+{ pkgs ? import <nixpkgs> {}
+}:
+pkgs.mkShell {
+  buildInputs = [ pkgs.pipenv ];
+  shellHook = ''
+    # set SOURCE_DATE_EPOCH so that we can use python wheels
+    SOURCE_DATE_EPOCH=$(date +%s)
+  '';
+}
diff --git a/tox.ini b/tox.ini
index 00880f9..df1e54e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,13 +1,13 @@
 [tox]
-envlist = py27, py33, py34, py35, py36, pypy, flake8, manifest
+envlist = py33, py34, py35, py36, py37, pypy, flake8, manifest
 
 [travis]
 python =
-    2.7: py27
     3.3: py33
     3.4: py34
     3.5: py35
     3.6: py36, flake8, manifest
+    3.7: py37
     pypy: pypy
 
 [testenv]
@@ -20,14 +20,12 @@ deps =
     pytest-mock
 
 [testenv:flake8]
-basepython = python2.7
 deps =
     flake8
 commands =
     flake8 make_lambda_package tests --max-line-length=120
 
 [testenv:manifest]
-basepython = python2.7
 deps = check-manifest
 skip_install = true
 commands = check-manifest {posargs}


Test patch:
diff --git a/tests/test_archive.py b/tests/test_archive.py
index 8b3e767..2c9f472 100644
--- a/tests/test_archive.py
+++ b/tests/test_archive.py
@@ -18,13 +18,13 @@ def test_repo_and_local_source_files(tmpdir):
         # this should not be added to the archive
         py.path.local().join('hello.txt').write('local')
 
-        archive.make_archive(paths, repo_source_files='hello.txt')
+        archive.make_archive(paths, None, repo_source_files='hello.txt')
 
         with ZipFile(paths.zip_path) as zipfile:
             with zipfile.open('hello.txt') as f:
                 assert f.read().decode('utf-8') == 'repo'
 
-        archive.make_archive(paths, local_source_files=[('hello.txt', 'dest.txt')])
+        archive.make_archive(paths, None, local_source_files=[('hello.txt', 'dest.txt')])
 
         with ZipFile(paths.zip_path) as zipfile:
             with zipfile.open('dest.txt') as f:
@@ -37,8 +37,9 @@ def test_deps_file(tmpdir):
         paths = fsutil.decide_paths(scm_source)
         fsutil.ensure_dirs(paths)
 
+        runtime = 'python2.7'
         site_packages_path = py.path.local(paths.build_dir).join(
-            'env', 'lib', 'python2.7', 'site-packages')
+            'env', 'lib', runtime, 'site-packages')
 
         fsutil.mkdir_p(str(site_packages_path))
 
@@ -47,7 +48,7 @@ def test_deps_file(tmpdir):
         deps_file = py.path.local('deps.txt')
         deps_file.write('\n'.join(['mypackage/hello.txt', '../ghost']))
 
-        archive.make_archive(paths, deps_file=str(deps_file))
+        archive.make_archive(paths, runtime, deps_file=str(deps_file))
 
         with ZipFile(paths.zip_path) as zipfile:
             assert len(zipfile.namelist()) == 1
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 6899547..4b33fb4 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -61,6 +61,31 @@ def test_with_all_source_options(tmpdir, runner):
             assert 'make_lambda_package/cli.py' in namelist
 
 
+def test_with_explicit_runtime(tmpdir, runner):
+    cwd = os.getcwd()
+    work_dir = tmpdir.join('deploy_package')
+    args = [
+        cwd,
+        '--repo-source-files', 'make_lambda_package/*.py',
+        '--requirements-file', 'tests/test-requirements.txt',
+        '--runtime', 'python3.6',
+        '--work-dir', str(work_dir),
+    ]
+
+    with tmpdir.as_cwd():
+        result = runner.invoke(cli.main, args)
+        assert not result.exception, result.output
+        assert result.exit_code == 0, result.output
+
+        zip_path = work_dir.join('dist', 'lambda-package.zip')
+        assert zip_path.isfile(), result.output
+
+        with ZipFile(str(zip_path)) as zipfile:
+            namelist = zipfile.namelist()
+            assert len(list(filter(lambda name: 'click' in name, namelist))) > 0
+            assert 'make_lambda_package/cli.py' in namelist
+
+
 def test_fail_if_no_docker_command(tmpdir, mocker, runner):
     with tmpdir.as_cwd():
         mocked_call = mocker.patch.object(subprocess, 'call', return_value=1)