Repository: ReproNim/reproman

Issue:
enable paramiko/fabric logging (into our handlers) if our debug level < DEBUG
In the light of #344 it is hard to figure out what is going on in fabric, so we need an easy way to enable its debug output.  I guess the easiest is just to direct their logger into our handlers

Discussion:

Patch:
diff --git a/reproman/resource/aws_ec2.py b/reproman/resource/aws_ec2.py
index 65de005..8eb376e 100644
--- a/reproman/resource/aws_ec2.py
+++ b/reproman/resource/aws_ec2.py
@@ -110,8 +110,15 @@ class AwsEc2(Resource):
     def create(self):
         """
         Create an EC2 instance.
-
-        Returns
+        
+        There are 2 yields for the AWS create method. The first yield occurs
+        immediately after the AWS service is sent the EC2 instance run command
+        so that the instance details can immediately be saved to the
+        inventory.yml file. The second yield occurs after the EC2 instance
+        has fully spun up and the "running" status is saved to the
+        inventory.yml file.
+
+        Yields
         -------
         dict : config and state parameters to capture in the inventory file
         """
@@ -162,6 +169,15 @@ class AwsEc2(Resource):
         # Save the EC2 Instance object.
         self._ec2_instance = self._ec2_resource.Instance(instances[0].id)
         self.id = self._ec2_instance.instance_id
+        self.status = self._ec2_instance.state['Name']
+
+        # Send initial info back to be saved in inventory file.
+        yield {
+            'id': self.id,
+            'status': self.status,
+            'key_name': self.key_name,
+            'key_filename': self.key_filename
+        }
 
         lgr.info("Waiting for EC2 instance %s to start running...", self.id)
         self._ec2_instance.wait_until_running(
@@ -172,7 +188,7 @@ class AwsEc2(Resource):
                 },
             ]
         )
-        lgr.info("EC2 instance %s to start running!", self.id)
+        lgr.info("EC2 instance %s is running!", self.id)
 
         lgr.info("Waiting for EC2 instance %s to complete initialization...",
                  self.id)
@@ -180,11 +196,8 @@ class AwsEc2(Resource):
         waiter.wait(InstanceIds=[self.id])
         lgr.info("EC2 instance %s initialized!", self.id)
         self.status = self._ec2_instance.state['Name']
-        return {
-            'id': self.id,
-            'status': self.status,
-            'key_name': self.key_name,
-            'key_filename': self.key_filename
+        yield {
+            'status': self.status
         }
 
     def delete(self):
@@ -278,7 +291,7 @@ Please enter a unique name to create a new key-pair or press [enter] to exit"""
             self.name,
             host=self._ec2_instance.public_ip_address,
             user=self.user,
-            key_filename=[self.key_filename]
+            key_filename=self.key_filename
         )
 
         return ssh.get_session(pty=pty, shared=shared)
diff --git a/reproman/resource/base.py b/reproman/resource/base.py
index 570854f..0fb4b76 100644
--- a/reproman/resource/base.py
+++ b/reproman/resource/base.py
@@ -332,10 +332,12 @@ class ResourceManager(object):
             config.update(backend_params)
         resource = self.factory(config)
         resource.connect()
-        resource_attrs = resource.create()
-        config.update(resource_attrs)
-        self.inventory[name] = config
-        self._save()
+        # Resource can yield and save inventory info as it needs to throughout
+        # the create process.
+        for resource_attrs in resource.create():
+            config.update(resource_attrs)
+            self.inventory[name] = config
+            self._save()
 
     def delete(self, resource):
         """Delete `resource` from the inventory.
diff --git a/reproman/resource/docker_container.py b/reproman/resource/docker_container.py
index c6667b4..bdeaafe 100644
--- a/reproman/resource/docker_container.py
+++ b/reproman/resource/docker_container.py
@@ -131,7 +131,7 @@ class DockerContainer(Resource):
         """
         Create a baseline Docker image and run it to create the container.
 
-        Returns
+        Yields
         -------
         dict : config parameters to capture in the inventory file
         """
@@ -166,7 +166,7 @@ class DockerContainer(Resource):
         self.id = self._container.get('Id')
         self._client.start(container=self.id)
         self.status = 'running'
-        return {
+        yield {
             'id': self.id,
             'status': self.status
         }
diff --git a/reproman/resource/shell.py b/reproman/resource/shell.py
index 7dd06e3..319e5ae 100644
--- a/reproman/resource/shell.py
+++ b/reproman/resource/shell.py
@@ -111,15 +111,14 @@ class Shell(Resource):
         """
         Create a running environment.
 
-        Parameters
-        ----------
-        name : string
-            Name identifier of the environment to be created.
+        Yields
+        -------
+        dict : config parameters to capture in the inventory file
         """
         # Generic logic to reside in Resource???
         if self.id is None:
             self.id = Resource._generate_id()
-        return {
+        yield {
             'id': self.id,
             'status': 'available'
         }
diff --git a/reproman/resource/singularity.py b/reproman/resource/singularity.py
index 279cd75..67683d6 100644
--- a/reproman/resource/singularity.py
+++ b/reproman/resource/singularity.py
@@ -71,7 +71,7 @@ class Singularity(Resource):
         """
         Create a container instance.
 
-        Returns
+        Yields
         -------
         dict : config parameters to capture in the inventory file
         """
@@ -90,7 +90,7 @@ class Singularity(Resource):
         # Update status
         self.id = "{name}-{pid}".format(**info)
         self.status = 'running'
-        return {
+        yield {
             'id': self.id,
             'status': self.status
         }
diff --git a/reproman/resource/ssh.py b/reproman/resource/ssh.py
index 98bad53..dd66bf6 100644
--- a/reproman/resource/ssh.py
+++ b/reproman/resource/ssh.py
@@ -13,10 +13,14 @@ import getpass
 import invoke
 import uuid
 from fabric import Connection
+from ..log import LoggerHelper
 from paramiko import AuthenticationException
 
 import logging
 lgr = logging.getLogger('reproman.resource.ssh')
+# Add Paramiko logging for log levels below DEBUG
+if lgr.getEffectiveLevel() < logging.DEBUG:
+    LoggerHelper("paramiko").get_initialized_logger()
 
 from .base import Resource
 from ..utils import attrib
@@ -59,12 +63,18 @@ class SSH(Resource):
             but do allow tests to authenticate by passing a password as
             a parameter to this method.
         """
+        # Convert key_filename to a list
+        # See: https://github.com/ReproNim/reproman/commit/3807f1287c39ea2393bae26803e6da8122ac5cff
+        key_filename = None
+        if self.key_filename:
+            key_filename = [self.key_filename]
+
         self._connection = Connection(
             self.host,
             user=self.user,
             port=self.port,
             connect_kwargs={
-                'key_filename': self.key_filename,
+                'key_filename': key_filename,
                 'password': password
             }
         )
@@ -97,14 +107,14 @@ class SSH(Resource):
         """
         Register the SSH connection to the reproman inventory registry.
 
-        Returns
+        Yields
         -------
-        dict : config and state parameters to capture in the inventory file
+        dict : config parameters to capture in the inventory file
         """
         if not self.id:
             self.id = str(uuid.uuid4())
         self.status = 'N/A'
-        return {
+        yield {
             'id': self.id,
             'status': self.status,
             'host': self.host,
diff --git a/reproman/utils.py b/reproman/utils.py
index 3e6c3cb..b112b5f 100644
--- a/reproman/utils.py
+++ b/reproman/utils.py
@@ -1366,4 +1366,23 @@ def command_as_string(command):
     return command
 
 
+def merge_dicts(ds):
+    """Convert an iterable of dictionaries.
+
+    In the case of key collisions, the last value wins.
+
+    Parameters
+    ----------
+    ds : iterable of dicts
+
+    Returns
+    -------
+    dict
+    """
+    merged = {}
+    for d in ds:
+        merged.update(d)
+    return merged
+
+
 lgr.log(5, "Done importing reproman.utils")
diff --git a/setup.py b/setup.py
index 7957680..dc2515e 100755
--- a/setup.py
+++ b/setup.py
@@ -155,12 +155,6 @@ setup(
     },
     cmdclass=cmdclass,
     package_data={
-        'reproman':
-            findsome(opj("distributions", "tests", "files"), {"yml", "yaml"}) +
-            findsome("examples", {"trig", "yml", "yaml"}) +
-            findsome(opj("formats", "tests", "files"), {"yml", "yaml"}) +
-            findsome(opj("interface", "tests"), {"yml", "yaml"}) +
-            findsome(opj("interface", "tests", "files"), {"yml", "yaml"}) +
-            findsome(opj("tests", "files"), {"cfg"})
+        'reproman': []
     }
 )


Test patch:
diff --git a/reproman/resource/tests/test_aws_ec2.py b/reproman/resource/tests/test_aws_ec2.py
index d304627..b4fad55 100644
--- a/reproman/resource/tests/test_aws_ec2.py
+++ b/reproman/resource/tests/test_aws_ec2.py
@@ -95,7 +95,7 @@ def test_awsec2_class():
 
         # Test creating an existing resource and catch the exception.
         try:
-            resource.create()
+            list(resource.create())
         except Exception as e:
             assert e.args[0] == "Instance 'i-00002777d52482d9c' already exists in AWS subscription"
 
@@ -104,7 +104,7 @@ def test_awsec2_class():
             instances=MagicMock(filter=lambda Filters: []),
             Instance=lambda id: MagicMock(
                 instance_id='i-11112777d52482d9c',
-                state={'Name': 'running'}
+                state={'Name': 'pending'}
             )
         )
         config = {
@@ -117,11 +117,16 @@ def test_awsec2_class():
         }
         resource = ResourceManager.factory(config)
         resource.connect()
-        results = resource.create()
-        assert results['id'] == 'i-11112777d52482d9c'
-        assert results['status'] == 'running'
+        # Test retrieving more than one yield from the create method
+        create_generator = resource.create()
+        result_1 = next(create_generator)
+        assert result_1['id'] == 'i-11112777d52482d9c'
+        assert result_1['status'] == 'pending'
+        resource._ec2_instance.state['Name'] = 'running'
+        result_2 = next(create_generator)
+        assert result_2['status'] == 'running'
         assert_in('EC2 instance i-11112777d52482d9c initialized!', log.lines)
-        assert_in('EC2 instance i-11112777d52482d9c to start running!', log.lines)
+        assert_in('EC2 instance i-11112777d52482d9c is running!', log.lines)
         assert_in('Waiting for EC2 instance i-11112777d52482d9c to complete initialization...', log.lines)
         assert_in('EC2 instance i-11112777d52482d9c initialized!', log.lines)
 
diff --git a/reproman/resource/tests/test_docker_container.py b/reproman/resource/tests/test_docker_container.py
index 3b7ba03..1e5cfb2 100644
--- a/reproman/resource/tests/test_docker_container.py
+++ b/reproman/resource/tests/test_docker_container.py
@@ -10,6 +10,7 @@
 import logging
 from mock import patch, MagicMock, call
 
+from ...utils import merge_dicts
 from ...utils import swallow_logs
 from ...tests.utils import assert_in, skip_if_no_docker_engine
 from ..base import ResourceManager
@@ -106,7 +107,7 @@ def test_dockercontainer_class():
 
         # Test creating an existing resource and catch the exception.
         try:
-            resource.create()
+            list(resource.create())
         except Exception as e:
             assert e.args[0] == "Container 'existing-test-resource' (ID 326b0fdfbf83) already exists in Docker"
 
@@ -118,7 +119,7 @@ def test_dockercontainer_class():
         }
         resource = ResourceManager.factory(config)
         resource.connect()
-        results = resource.create()
+        results = merge_dicts(resource.create())
         assert results['id'] == '18b31b30e3a5'
         assert results['status'] == 'running'
         assert_in('status 1 progress 1', log.lines)
diff --git a/reproman/resource/tests/test_session.py b/reproman/resource/tests/test_session.py
index eba9f05..73233cd 100644
--- a/reproman/resource/tests/test_session.py
+++ b/reproman/resource/tests/test_session.py
@@ -244,7 +244,7 @@ def resource_session(request):
         name = str(uuid.uuid4().hex)[:11]
         resource = Singularity(name=name, image='docker://python:2.7')
         resource.connect()
-        resource.create()
+        list(resource.create())
         yield request.param(name)
         resource.delete()
     else:
diff --git a/reproman/resource/tests/test_shell.py b/reproman/resource/tests/test_shell.py
index ff99f2d..394a086 100644
--- a/reproman/resource/tests/test_shell.py
+++ b/reproman/resource/tests/test_shell.py
@@ -15,6 +15,7 @@ import tempfile
 from pytest import raises
 from mock import patch, call
 
+from ...utils import merge_dicts
 from ...utils import swallow_logs
 from ...tests.utils import assert_in
 from ..base import ResourceManager
@@ -135,7 +136,7 @@ def test_shell_resource():
     }
     resource = ResourceManager.factory(config)
 
-    status = resource.create()
+    status = merge_dicts(resource.create())
     assert re.match('\w{8}-\w{4}-\w{4}-\w{4}-\w{12}$', status['id']) is not None
 
     assert type(resource.connect()) == Shell
diff --git a/reproman/resource/tests/test_singularity.py b/reproman/resource/tests/test_singularity.py
index 5d06ba0..243c701 100644
--- a/reproman/resource/tests/test_singularity.py
+++ b/reproman/resource/tests/test_singularity.py
@@ -42,7 +42,7 @@ def test_singularity_resource_class():
         resource.connect()
         assert resource.id is None
         assert resource.status is None
-        resource.create()
+        list(resource.create())
         assert resource.id.startswith('foo-')
         assert resource.status == 'running'
 
@@ -52,7 +52,7 @@ def test_singularity_resource_class():
         resource_duplicate.connect()
         assert resource_duplicate.id.startswith('foo-')
         assert resource_duplicate.status == 'running'
-        resource_duplicate.create()
+        list(resource_duplicate.create())
         assert_in('Resource foo already exists.', log.lines)
 
         # Test retrieving instance info.
diff --git a/reproman/resource/tests/test_ssh.py b/reproman/resource/tests/test_ssh.py
index 224bf61..a9f1c7b 100644
--- a/reproman/resource/tests/test_ssh.py
+++ b/reproman/resource/tests/test_ssh.py
@@ -17,6 +17,7 @@ import six
 import uuid
 from pytest import raises
 
+from ...utils import merge_dicts
 from ...utils import swallow_logs
 from ...tests.utils import assert_in, skip_ssh
 from ..base import ResourceManager
@@ -57,7 +58,7 @@ def test_ssh_class(setup_ssh, resource_test_dir):
             **setup_ssh['custom']
         )
         resource = ResourceManager.factory(config)
-        updated_config = resource.create()
+        updated_config = merge_dicts(resource.create())
         config.update(updated_config)
         assert re.match('\w{8}-\w{4}-\w{4}-\w{4}-\w{12}$',
                         resource.id) is not None
diff --git a/reproman/tests/test_utils.py b/reproman/tests/test_utils.py
index 8f3fd6e..b0eaea5 100644
--- a/reproman/tests/test_utils.py
+++ b/reproman/tests/test_utils.py
@@ -48,6 +48,7 @@ from ..utils import to_unicode
 from ..utils import generate_unique_name
 from ..utils import PathRoot, is_subpath
 from ..utils import parse_semantic_version
+from ..utils import merge_dicts
 
 from .utils import ok_, eq_, assert_false, assert_equal, assert_true
 
@@ -548,6 +549,15 @@ def test_parse_semantic_version():
         parse_semantic_version("1.2")
 
 
+def test_merge_dicts():
+    assert merge_dicts([]) == {}
+    assert merge_dicts([{1: 1}]) == {1: 1}
+    assert merge_dicts([{1: 1}, {2: 2}]) == {1: 1, 2: 2}
+    assert merge_dicts([{1: 1}, {2: 2}, {1: 3}]) == {1: 3, 2: 2}
+    assert merge_dicts(iter([{1: 1}, {2: 2}, {1: 3}])) == {1: 3, 2: 2}
+    assert merge_dicts([{1: 1}, {2: 2}, {1: 3}]) == {1: 3, 2: 2}
+
+
 def test_line_profile():
     pytest.importorskip("line_profiler")