Repository: rackerlabs/lambda-uploader

Issue:
Option zip and upload only the folder contents
Right now, uploading grabs a bunch of stuff I don't need (git, PyGitHub, etc). This creates a fairly large zip file, hence, larger lambda storage.

Not being strong with Python, I'm guessing the extra stuff relates to using virtualenv. I don't need all those dependencies, and would prefer to pip install to the local directory. Could this be added as a new switch? Or is there something I'm missing and there's some other way for me to skip them?

Discussion:

Patch:
diff --git a/README.md b/README.md
index 332bb35..bf0a3bb 100644
--- a/README.md
+++ b/README.md
@@ -29,11 +29,7 @@ Example lambda.json file:
   "handler": "function.lambda_handler",
   "role": "arn:aws:iam::00000000000:role/lambda_basic_execution",
   "requirements": ["pygithub"],
-  "ignore": [
-    "circle.yml",
-    ".git",
-    "*.pyc"
-  ],
+  "ignore": ["circle.yml"],
   "timeout": 30,
   "memory": 512
 }
@@ -57,6 +53,11 @@ To specify an alternative, prexisting virtualenv use the `--virtualenv` paramete
 lambda-uploader --virtualenv=~/.virtualenv/my_custom_virtualenv
 ```
 
+To omit using a virtualenv use the `--no-virtualenv` parameter.
+```shell
+lambda-uploader --no-virtualenv
+```
+
 If you would prefer to upload another way you can tell the uploader to ignore the upload.
 This will create a package and leave it in the project directory.
 ```shell
diff --git a/README.rst b/README.rst
index eaa1676..a9bf569 100644
--- a/README.rst
+++ b/README.rst
@@ -42,11 +42,7 @@ Example lambda.json file:
       "handler": "function.lambda_handler",
       "role": "arn:aws:iam::00000000000:role/lambda_basic_execution",
       "requirements": ["pygithub"],
-      "ignore": [
-        "circle.yml",
-        ".git",
-        "*.pyc"
-      ],
+      "ignore": ["circle.yml"],
       "timeout": 30,
       "memory": 512
     }
@@ -75,6 +71,12 @@ To specify an alternative, prexisting virtualenv use the
 
     lambda-uploader --virtualenv=~/.virtualenv/my_custom_virtualenv
 
+To omit using a virtualenv use the ``--no-virtualenv`` parameter.
+
+.. code:: shell
+
+    lambda-uploader --no-virtualenv
+
 If you would prefer to upload another way you can tell the uploader to
 ignore the upload. This will create a package and leave it in the
 project directory.
diff --git a/example/lambda.json b/example/lambda.json
index bdf786a..7148a5b 100644
--- a/example/lambda.json
+++ b/example/lambda.json
@@ -5,11 +5,7 @@
   "handler": "function.lambda_handler",
   "role": "arn:aws:iam::00000000000:role/lambda_basic_execution",
   "requirements": ["Jinja2==2.8"],
-  "ignore": [
-    "circle.yml",
-    ".git",
-    "*.pyc"
-  ],
+  "ignore": ["circle.yml"],
   "timeout": 30,
   "memory": 512
 }
diff --git a/lambda_uploader/package.py b/lambda_uploader/package.py
index 1bb0089..ccbe79e 100644
--- a/lambda_uploader/package.py
+++ b/lambda_uploader/package.py
@@ -27,37 +27,24 @@ ZIPFILE_NAME = 'lambda_function.zip'
 
 
 def build_package(path, requirements, virtualenv=None, ignore=[]):
-    pkg = Package(path, virtualenv)
+    pkg = Package(path, virtualenv, requirements)
 
     pkg.clean_workspace()
     pkg.clean_zipfile()
     pkg.prepare_workspace()
-    if virtualenv:
-        if not os.path.isdir(virtualenv):
-            raise Exception("supplied virtualenv %s not found" % virtualenv)
-        LOG.info("Using existing virtualenv found in %s" % virtualenv)
-    else:
-        LOG.info('Building new virtualenv and installing requirements')
-        pkg.prepare_virtualenv()
-        pkg.install_requirements(requirements)
+    pkg.prepare_virtualenv()
     pkg.package(ignore)
     return pkg
 
 
 class Package(object):
-    def __init__(self, path, virtualenv=None):
+    def __init__(self, path, virtualenv=None, requirements=[]):
         self._path = path
         self._temp_workspace = os.path.join(path,
                                             TEMP_WORKSPACE_NAME)
         self.zip_file = os.path.join(path, ZIPFILE_NAME)
-
-        if virtualenv:
-            self._pkg_venv = virtualenv
-        else:
-            self._pkg_venv = os.path.join(self._temp_workspace, 'venv')
-            self._venv_pip = 'bin/pip'
-            if sys.platform == 'win32' or sys.platform == 'cygwin':
-                self._venv_pip = 'Scripts\pip.exe'
+        self._virtualenv = virtualenv
+        self._requirements = requirements
 
     def clean_workspace(self):
         if os.path.isdir(self._temp_workspace):
@@ -72,21 +59,59 @@ class Package(object):
         os.mkdir(self._temp_workspace)
 
     def prepare_virtualenv(self):
-        proc = Popen(["virtualenv", self._pkg_venv], stdout=PIPE, stderr=PIPE)
-        stdout, stderr = proc.communicate()
-        LOG.debug("Virtualenv stdout: %s" % stdout)
-        LOG.debug("Virtualenv stderr: %s" % stderr)
+        requirements_exist = \
+            self._requirements or os.path.isfile("requirements.txt")
+        if self._virtualenv and self._virtualenv is not False:
+            if not os.path.isdir(self._virtualenv):
+                raise Exception("virtualenv %s not found" % self._virtualenv)
+            LOG.info("Using existing virtualenv at %s" % self._virtualenv)
+
+            # use supplied virtualenv path
+            self._pkg_venv = self._virtualenv
+        elif self._virtualenv is None and requirements_exist:
+            LOG.info('Building new virtualenv and installing requirements')
+            self.build_new_virtualenv()
+            self.install_requirements()
+        elif self._virtualenv is None and not requirements_exist:
+            LOG.info('No requirements found, so no virtualenv will be made')
+            self._pkg_venv = False
+        elif self._virtualenv is False:
+            LOG.info('Virtualenv has been omitted by supplied flag')
+            self._pkg_venv = False
+        else:
+            raise Exception('Cannot determine what to do about virtualenv')
 
-        if proc.returncode is not 0:
-            raise Exception('virtualenv returned unsuccessfully')
+    def build_new_virtualenv(self):
+        if self._virtualenv is None:
+            # virtualenv was "None" which means "do default"
+            self._pkg_venv = os.path.join(self._temp_workspace, 'venv')
+            self._venv_pip = 'bin/pip'
+            if sys.platform == 'win32' or sys.platform == 'cygwin':
+                self._venv_pip = 'Scripts\pip.exe'
+
+            proc = Popen(["virtualenv", self._pkg_venv],
+                         stdout=PIPE, stderr=PIPE)
+            stdout, stderr = proc.communicate()
+            LOG.debug("Virtualenv stdout: %s" % stdout)
+            LOG.debug("Virtualenv stderr: %s" % stderr)
+
+            if proc.returncode is not 0:
+                raise Exception('virtualenv returned unsuccessfully')
+
+        else:
+            raise Exception('cannot build a new virtualenv when asked to omit')
+
+    def install_requirements(self):
+        if not hasattr(self, '_pkg_venv'):
+            err = 'Must call build_new_virtualenv before install_requirements'
+            raise Exception(err)
 
-    def install_requirements(self, requirements):
         cmd = None
-        if requirements:
+        if self._requirements:
             LOG.debug("Installing requirements found %s in config"
-                      % requirements)
+                      % self._requirements)
             cmd = [os.path.join(self._pkg_venv, self._venv_pip),
-                   'install'] + requirements
+                   'install'] + self._requirements
 
         elif os.path.isfile("requirements.txt"):
             # Pip install
@@ -109,18 +134,19 @@ class Package(object):
         # Copy site packages into package base
         LOG.info('Copying site packages')
 
-        site_packages = 'lib/python2.7/site-packages'
-        lib64_site_packages = 'lib64/python2.7/site-packages'
-        if sys.platform == 'win32' or sys.platform == 'cygwin':
-            lib64_site_packages = 'lib64\\site-packages'
-            site_packages = 'lib\\site-packages'
-
-        utils.copy_tree(os.path.join(self._pkg_venv, site_packages),
-                        package)
-        lib64_path = os.path.join(self._pkg_venv, lib64_site_packages)
-        if not os.path.islink(lib64_path):
-            LOG.info('Copying lib64 site packages')
-            utils.copy_tree(lib64_path, package)
+        if hasattr(self, '_pkg_venv') and self._pkg_venv:
+            site_packages = 'lib/python2.7/site-packages'
+            lib64_site_packages = 'lib64/python2.7/site-packages'
+            if sys.platform == 'win32' or sys.platform == 'cygwin':
+                lib64_site_packages = 'lib64\\site-packages'
+                site_packages = 'lib\\site-packages'
+
+            utils.copy_tree(os.path.join(self._pkg_venv, site_packages),
+                            package)
+            lib64_path = os.path.join(self._pkg_venv, lib64_site_packages)
+            if not os.path.islink(lib64_path):
+                LOG.info('Copying lib64 site packages')
+                utils.copy_tree(lib64_path, package)
 
         # Append the temp workspace to the ignore list
         ignore.append("^%s/*" % self._temp_workspace)
diff --git a/lambda_uploader/shell.py b/lambda_uploader/shell.py
index c36fe21..a62bdb5 100644
--- a/lambda_uploader/shell.py
+++ b/lambda_uploader/shell.py
@@ -49,9 +49,19 @@ def _execute(args):
 
     cfg = config.Config(pth, args.config, role=args.role)
 
+    if args.no_virtualenv:
+        # specified flag to omit entirely
+        venv = False
+    elif args.virtualenv:
+        # specified a custom virtualenv
+        venv = args.virtualenv
+    else:
+        # build and include virtualenv, the default
+        venv = None
+
     _print('Building Package')
     pkg = package.build_package(pth, cfg.requirements,
-                                args.virtualenv, cfg.ignore)
+                                venv, cfg.ignore)
 
     if not args.no_clean:
         pkg.clean_workspace()
@@ -101,6 +111,10 @@ def main(arv=None):
     parser.add_argument('--virtualenv', '-e',
                         help='use specified virtualenv instead of making one',
                         default=None)
+    parser.add_argument('--no-virtualenv', dest='no_virtualenv',
+                        action='store_const',
+                        help='do not create or include a virtualenv at all',
+                        const=True)
     parser.add_argument('--role', dest='role',
                         default=getenv('LAMBDA_UPLOADER_ROLE'),
                         help=('IAM role to assign the lambda function, '


Test patch:
diff --git a/test/test_package.py b/test/test_package.py
index 1a64572..a293d4f 100644
--- a/test/test_package.py
+++ b/test/test_package.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import pytest
 
 from shutil import rmtree
 from os import path
@@ -53,9 +54,9 @@ def test_install_requirements():
     temp_workspace = path.join(TESTING_TEMP_DIR,
                                package.TEMP_WORKSPACE_NAME)
 
-    pkg = package.Package(TESTING_TEMP_DIR)
-    # pkg.prepare_workspace()
-    pkg.install_requirements(reqs)
+    pkg = package.Package(TESTING_TEMP_DIR, requirements=reqs)
+    pkg.prepare_virtualenv()
+
     site_packages = path.join(temp_workspace,
                               'venv/lib/python2.7/site-packages')
     if sys.platform == 'win32' or sys.platform == 'cygwin':
@@ -64,9 +65,39 @@ def test_install_requirements():
     assert path.isdir(path.join(site_packages, '_pytest'))
 
 
+def test_default_virtualenv():
+    temp_workspace = path.join(TESTING_TEMP_DIR,
+                               package.TEMP_WORKSPACE_NAME)
+    pkg = package.Package(TESTING_TEMP_DIR)
+    pkg.prepare_virtualenv()
+    # ensure we picked a real venv path if using default behavior
+    assert pkg._pkg_venv == ("%s/venv" % temp_workspace)
+
+
 def test_existing_virtualenv():
+    venv_dir = "virtualenv_test"
+    temp_virtualenv = path.join(TESTING_TEMP_DIR, venv_dir)
+    os.mkdir(temp_virtualenv)
+
+    pkg = package.Package(TESTING_TEMP_DIR, temp_virtualenv)
+    pkg.prepare_virtualenv()
+
+    assert pkg._pkg_venv == temp_virtualenv
+
+
+def test_bad_existing_virtualenv():
     pkg = package.Package(TESTING_TEMP_DIR, 'abc')
-    assert pkg._pkg_venv == 'abc'
+    with pytest.raises(Exception):
+        pkg.prepare_virtualenv()
+
+
+def test_omit_virtualenv():
+    pkg = package.Package(TESTING_TEMP_DIR, False)
+    pkg.prepare_virtualenv()
+    assert pkg._pkg_venv is False
+
+    with pytest.raises(Exception):
+        pkg.build_new_virtualenv()
 
 
 def test_package():