Merge pull request #7 from rackerlabs/refactor_package

Refactor Package

Author: martinb3

circle.yml

@@ -2,12 +2,9 @@ machine:
   python:
     version: 2.7
 
-dependencies:
-  post:
-    - rm -r ~/virtualenvs
-
 test:
   pre:
     - pip install -r test-requirements.txt
   override:
     - flake8 .
+    - py.test

lambda_uploader/package.py

@@ -25,105 +25,116 @@
 ZIPFILE_NAME = 'lambda_function.zip'
 
 
-def cleanup(path):
-    temp_workspace = os.path.join(path, TEMP_WORKSPACE_NAME)
-    if os.path.isdir(temp_workspace):
-        shutil.rmtree(temp_workspace)
-    zip_file = os.path.join(path, ZIPFILE_NAME)
-    if os.path.isfile(zip_file):
-        os.remove(zip_file)
+def build_package(path, requirements):
+    pkg = Package(path)
 
+    pkg.clean_workspace()
+    pkg.clean_zipfile()
+    pkg.prepare_workspace()
+    pkg.install_requirements(requirements)
+    pkg.package()
+    return pkg
 
-def build_package(path, requirements):
-    temp_workspace = os.path.join(path, TEMP_WORKSPACE_NAME)
-    # Calling cleanup first to cover a previous failed run
-    cleanup(path)
-
-    # Setup temporary workspace
-    os.mkdir(temp_workspace)
-
-    pkg_venv = os.path.join(temp_workspace, 'venv')
-    proc = Popen(["virtualenv", pkg_venv], stdout=PIPE, stderr=PIPE)
-    stdout, stderr = proc.communicate()
-    LOG.debug("Virtualenv stdout: %s" % stdout)
-    LOG.debug("Virtualenv stderr: %s" % stderr)
-
-    if proc.returncode is not 0:
-        raise Exception('virtualenv returned unsuccessfully')
-
-    cmd = None
-    if requirements:
-        LOG.debug("Installing requirements found %s in config" % requirements)
-        cmd = [os.path.join(pkg_venv, _venv_pip()),
-               'install', " ".join(requirements)]
-
-    elif os.path.isfile("requirements.txt"):
-        # Pip install
-        LOG.debug("Installing requirements from requirements.txt file")
-        cmd = [os.path.join(pkg_venv, _venv_pip()),
-               "install", "-r", "requirements.txt"]
-
-    if cmd is not None:
-        prc = Popen(cmd, stdout=PIPE, stderr=PIPE)
-        stdout, stderr = prc.communicate()
-        LOG.debug("Pip stdout: %s" % stdout)
-        LOG.debug("Pip stderr: %s" % stderr)
+
+class Package(object):
+    def __init__(self, path):
+        self._path = path
+        self._temp_workspace = os.path.join(path,
+                                            TEMP_WORKSPACE_NAME)
+        self.zip_file = os.path.join(path, ZIPFILE_NAME)
+
+        self._pkg_venv = os.path.join(self._temp_workspace, 'venv')
+        self._venv_pip = 'bin/pip'
+        if sys.platform == 'win32' or sys.platform == 'cygwin':
+            self._venv_pip = 'Scripts\pip.exe'
+
+    def clean_workspace(self):
+        if os.path.isdir(self._temp_workspace):
+            shutil.rmtree(self._temp_workspace)
+
+    def clean_zipfile(self):
+        if os.path.isfile(self.zip_file):
+            os.remove(self.zip_file)
+
+    def prepare_workspace(self):
+        # Setup temporary workspace
+        os.mkdir(self._temp_workspace)
+
+        proc = Popen(["virtualenv", self._pkg_venv], stdout=PIPE, stderr=PIPE)
+        stdout, stderr = proc.communicate()
+        LOG.debug("Virtualenv stdout: %s" % stdout)
+        LOG.debug("Virtualenv stderr: %s" % stderr)
 
         if proc.returncode is not 0:
-            raise Exception('pip returned unsuccessfully')
-    package = os.path.join(temp_workspace, 'lambda_package')
-
-    # Copy site packages into package base
-    LOG.info('Copying site packages')
-
-    site_packages = 'lib/python2.7/site-packages'
-    if sys.platform == 'win32' or sys.platform == 'cygwin':
-        site_packages = 'lib\\site-packages'
-
-    shutil.copytree(os.path.join(pkg_venv, site_packages),
-                    package)
-    _copy_src_files(path, package)
-    _create_zip(package, path)
-
-
-def _venv_pip():
-    if sys.platform == 'win32' or sys.platform == 'cygwin':
-        return 'Scripts\pip.exe'
-    else:
-        return 'bin/pip'
-
-
-def _copy_src_files(src, package):
-    LOG.info('Copying source files')
-    # Re-create cwd directory structure
-    for root, subdirs, files in os.walk(src):
-        for subdir in subdirs:
-            subdir_path = os.path.join(root, subdir)
-            if TEMP_WORKSPACE_NAME in subdir_path:
-                continue
-            fpath = os.path.join(package, subdir)
-            LOG.debug("Creating directory %s" % fpath)
-            os.mkdir(fpath)
-
-        for filename in files:
-            path = os.path.join(root, filename)
-            if TEMP_WORKSPACE_NAME in path:
-                continue
-
-            shutil.copy(path, package)
-
-
-def _create_zip(src, dest):
-    zfile = os.path.join(dest, ZIPFILE_NAME)
-    LOG.info('Creating zipfile')
-    zf = zipfile.ZipFile(zfile, "w", zipfile.ZIP_DEFLATED)
-    abs_src = os.path.abspath(src)
-    for root, _, files in os.walk(src):
-        for filename in files:
-            absname = os.path.abspath(os.path.join(root, filename))
-            arcname = absname[len(abs_src) + 1:]
-            LOG.debug('Zipping %s as %s'
-                      % (os.path.join(root, filename), arcname))
-
-            zf.write(absname, arcname)
-    zf.close()
+            raise Exception('virtualenv returned unsuccessfully')
+
+    def install_requirements(self, requirements):
+        cmd = None
+        if requirements:
+            LOG.debug("Installing requirements found %s in config"
+                      % requirements)
+            cmd = [os.path.join(self._pkg_venv, self._venv_pip),
+                   'install', " ".join(requirements)]
+
+        elif os.path.isfile("requirements.txt"):
+            # Pip install
+            LOG.debug("Installing requirements from requirements.txt file")
+            cmd = [os.path.join(self._pkg_venv, self._venv_pip),
+                   "install", "-r", "requirements.txt"]
+
+        if cmd is not None:
+            prc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+            stdout, stderr = prc.communicate()
+            LOG.debug("Pip stdout: %s" % stdout)
+            LOG.debug("Pip stderr: %s" % stderr)
+
+            if prc.returncode is not 0:
+                raise Exception('pip returned unsuccessfully')
+
+    def package(self):
+        package = os.path.join(self._temp_workspace, 'lambda_package')
+
+        # Copy site packages into package base
+        LOG.info('Copying site packages')
+        site_packages = 'lib/python2.7/site-packages'
+        if sys.platform == 'win32' or sys.platform == 'cygwin':
+            site_packages = 'lib\\site-packages'
+
+        shutil.copytree(os.path.join(self._pkg_venv, site_packages),
+                        package)
+        self._copy_src_files(package)
+        self._create_zip(package)
+
+    def _copy_src_files(self, package):
+        LOG.info('Copying source files')
+        # Re-create cwd directory structure
+        for root, subdirs, files in os.walk(self._path):
+            for subdir in subdirs:
+                subdir_path = os.path.join(root, subdir)
+                if TEMP_WORKSPACE_NAME in subdir_path:
+                    continue
+                fpath = os.path.join(package, subdir)
+                LOG.debug("Creating directory %s" % fpath)
+                os.mkdir(fpath)
+
+            for filename in files:
+                path = os.path.join(root, filename)
+                if TEMP_WORKSPACE_NAME in path:
+                    continue
+
+                shutil.copy(path, package)
+
+    def _create_zip(self, src):
+        zfile = os.path.join(self._path, ZIPFILE_NAME)
+        LOG.info('Creating zipfile')
+        zf = zipfile.ZipFile(zfile, "w", zipfile.ZIP_DEFLATED)
+        abs_src = os.path.abspath(src)
+        for root, _, files in os.walk(src):
+            for filename in files:
+                absname = os.path.abspath(os.path.join(root, filename))
+                arcname = absname[len(abs_src) + 1:]
+                LOG.debug('Zipping %s as %s'
+                          % (os.path.join(root, filename), arcname))
+
+                zf.write(absname, arcname)
+        zf.close()

lambda_uploader/shell.py

@@ -49,13 +49,15 @@ def _execute(args):
 
     cfg = config.Config(pth)
     _print('Building Package')
-    package.build_package(pth, cfg.requirements)
-    if not args.no_upload:
-        _print('Uploading Package')
-        uploader.upload_package(pth, cfg)
+    pkg = package.build_package(pth, cfg.requirements)
 
     if not args.no_clean:
-        package.cleanup(pth)
+        pkg.clean_workspace()
+
+    if not args.no_upload:
+        _print('Uploading Package')
+        uploader.upload_package(pkg, cfg)
+        pkg.clean_zipfile()
 
     _print('Fin')
 
@@ -74,7 +76,7 @@ def main(arv=None):
 
     parser.add_argument('--no-clean', dest='no_clean',
                         action='store_const',
-                        help='dont cleanup any files after uploading',
+                        help='dont cleanup the temporary workspace',
                         const=True)
     parser.add_argument('function_dir', default=getcwd(), nargs='?',
                         help='lambda function directory')

lambda_uploader/uploader.py

@@ -15,15 +15,11 @@
 import boto3
 import logging
 
-from os import path
-from lambda_uploader import package
-
 LOG = logging.getLogger(__name__)
 
 
-def upload_package(function_dir, config):
-    zfile = path.join(function_dir, package.ZIPFILE_NAME)
-    with open(zfile, "rb") as fil:
+def upload_package(pkg, config):
+    with open(pkg.zip_file, "rb") as fil:
         zip_file = fil.read()
 
     client = boto3.client('lambda', region_name=config.region)

test-requirements.txt

@@ -1,2 +1,3 @@
 pylint==1.4.1
 flake8==2.3.0
+pytest==2.8.2

test/test_package.py

@@ -0,0 +1,69 @@
+import os
+import sys
+
+from shutil import rmtree
+from os import path
+from lambda_uploader import package
+
+TESTING_TEMP_DIR = '.testing_temp'
+
+
+def setup_module(module):
+    print('calling setup')
+    os.mkdir(TESTING_TEMP_DIR)
+
+
+def teardown_module(module):
+    print('calling teardown')
+    rmtree(TESTING_TEMP_DIR)
+
+
+def test_package_zip_location():
+    pkg = package.Package(TESTING_TEMP_DIR)
+    assert pkg.zip_file == '.testing_temp/lambda_function.zip'
+
+
+def test_package_clean_workspace():
+    temp_workspace = path.join(TESTING_TEMP_DIR,
+                               '.lambda_package')
+    os.mkdir(temp_workspace)
+
+    pkg = package.Package(TESTING_TEMP_DIR)
+    pkg.clean_workspace()
+    assert path.isdir(temp_workspace) == False
+
+
+def test_prepare_workspace():
+    temp_workspace = path.join(TESTING_TEMP_DIR,
+                               '.lambda_package')
+
+    pkg = package.Package(TESTING_TEMP_DIR)
+    pkg.prepare_workspace()
+    assert path.isdir(temp_workspace)
+    assert path.isdir(path.join(temp_workspace, 'venv'))
+    if sys.platform == 'win32' or sys.platform == 'cygwin':
+        assert path.isfile(path.join(temp_workspace, "venv\\Scripts\\pip.exe"))
+    else:
+        assert path.isfile(path.join(temp_workspace, 'venv/bin/pip'))
+
+
+def test_install_requirements():
+    reqs = ['pytest']
+    temp_workspace = path.join(TESTING_TEMP_DIR,
+                               '.lambda_package')
+
+    pkg = package.Package(TESTING_TEMP_DIR)
+    # pkg.prepare_workspace()
+    pkg.install_requirements(reqs)
+    site_packages = path.join(temp_workspace,
+                              'venv/lib/python2.7/site-packages')
+    if sys.platform == 'win32' or sys.platform == 'cygwin':
+        site_packages = path.join(temp_workspace, "venv\\lib\\site-packages")
+
+    assert path.isdir(path.join(site_packages, '_pytest'))
+
+
+def test_package():
+    pkg = package.Package(TESTING_TEMP_DIR)
+    pkg.package()
+    assert path.isfile(path.join(TESTING_TEMP_DIR, 'lambda_function.zip'))

tox.ini

@@ -0,0 +1,3 @@
+[pytest]
+norecursedirs = venv
+testpaths = test