Repository: cloudant/python-cloudant

Issue:
Support IAM authentication in replication documents.
_Parent Issue: https://github.com/cloudant/cloudant-sync/issues/31_

Currently `db.creds['basic_auth']` returns `None` when using an IAM client . As a result, all replication documents created using IAM clients _don't_ contain source/target database credentials.

_Work Required:_
* Expose the IAM key via the client session ([here](https://github.com/cloudant/python-cloudant/blob/2.7.0/src/cloudant/_client_session.py#L194)).
* Rework the [Replicator](https://github.com/cloudant/python-cloudant/blob/2.7.0/src/cloudant/replicator.py#L77-L93) class to correctly inject the IAM key into the replication document (when databases are backed by an IAM client).
* Enable replicator tests for IAM clients (currently disabled).
* Add more unit tests as appropriate.

Discussion:
emlaver: Update copyright for `_client_session.py`

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 27f5b9b..ab805d4 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,7 +1,8 @@
 # Unreleased
 
-- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x.
 - [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database.
+- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x.
+- [NEW] Support IAM authentication in replication documents.
 - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method.
 - [IMPROVED] Updated Travis CI and unit tests to run against CouchDB 2.1.1.
 
diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py
index 93df457..0e8fd38 100644
--- a/src/cloudant/_client_session.py
+++ b/src/cloudant/_client_session.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2015, 2017 IBM Corp. All rights reserved.
+# Copyright (c) 2015, 2018 IBM Corp. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -195,6 +195,15 @@ class IAMSession(ClientSession):
         self._token_url = os.environ.get(
             'IAM_TOKEN_URL', 'https://iam.bluemix.net/identity/token')
 
+    @property
+    def get_api_key(self):
+        """
+        Get IAM API key.
+
+        :return: IAM API key.
+        """
+        return self._api_key
+
     def login(self):
         """
         Perform IAM cookie based user login.
diff --git a/src/cloudant/client.py b/src/cloudant/client.py
index 59d25b2..de88112 100755
--- a/src/cloudant/client.py
+++ b/src/cloudant/client.py
@@ -98,6 +98,15 @@ class CouchDB(dict):
         if connect_to_couch and self._DATABASE_CLASS == CouchDatabase:
             self.connect()
 
+    @property
+    def is_iam_authenticated(self):
+        """
+        Show if a client has authenticated using an IAM API key.
+
+        :return: True if client is IAM authenticated. False otherwise.
+        """
+        return self._use_iam
+
     def connect(self):
         """
         Starts up an authentication session for the client using cookie
@@ -107,10 +116,12 @@ class CouchDB(dict):
             self.session_logout()
 
         if self.admin_party:
+            self._use_iam = False
             self.r_session = ClientSession(
                 timeout=self._timeout
             )
         elif self._use_basic_auth:
+            self._use_iam = False
             self.r_session = BasicSession(
                 self._user,
                 self._auth_token,
diff --git a/src/cloudant/replicator.py b/src/cloudant/replicator.py
index c9b6a40..7f24407 100644
--- a/src/cloudant/replicator.py
+++ b/src/cloudant/replicator.py
@@ -51,8 +51,7 @@ class Replicator(object):
         :param str repl_id: Optional replication id.  Generated internally if
             not explicitly set.
         :param dict user_ctx: Optional user to act as.  Composed internally
-            if not explicitly set and not in CouchDB Admin Party
-            mode.
+            if not explicitly set.
         :param bool create_target: Specifies whether or not to
             create the target, if it does not already exist.
         :param bool continuous: If set to True then the replication will be
@@ -60,32 +59,50 @@ class Replicator(object):
 
         :returns: Replication document as a Document instance
         """
+        if source_db is None:
+            raise CloudantReplicatorException(101)
+
+        if target_db is None:
+            raise CloudantReplicatorException(102)
 
         data = dict(
             _id=repl_id if repl_id else str(uuid.uuid4()),
             **kwargs
         )
 
-        if source_db is None:
-            raise CloudantReplicatorException(101)
+        # replication source
+
         data['source'] = {'url': source_db.database_url}
-        if not source_db.admin_party:
-            data['source'].update(
-                {'headers': {'Authorization': source_db.creds['basic_auth']}}
-            )
+        if source_db.admin_party:
+            pass  # no credentials required
+        elif source_db.client.is_iam_authenticated:
+            data['source'].update({'auth': {
+                'iam': {'api_key': source_db.client.r_session.get_api_key}
+            }})
+        else:
+            data['source'].update({'headers': {
+                'Authorization': source_db.creds['basic_auth']
+            }})
+
+        # replication target
 
-        if target_db is None:
-            raise CloudantReplicatorException(102)
         data['target'] = {'url': target_db.database_url}
-        if not target_db.admin_party:
-            data['target'].update(
-                {'headers': {'Authorization': target_db.creds['basic_auth']}}
-            )
-
-        if not data.get('user_ctx'):
-            if (target_db and not target_db.admin_party or
-                    self.database.creds):
-                data['user_ctx'] = self.database.creds['user_ctx']
+        if target_db.admin_party:
+            pass  # no credentials required
+        elif target_db.client.is_iam_authenticated:
+            data['target'].update({'auth': {
+                'iam': {'api_key': target_db.client.r_session.get_api_key}
+            }})
+        else:
+            data['target'].update({'headers': {
+                'Authorization': target_db.creds['basic_auth']
+            }})
+
+        # add user context delegation
+
+        if not data.get('user_ctx') and self.database.creds and \
+            self.database.creds.get('user_ctx'):
+            data['user_ctx'] = self.database.creds['user_ctx']
 
         return self.database.create_document(data, throw_on_exists=True)
 


Test patch:
diff --git a/tests/unit/replicator_mock_tests.py b/tests/unit/replicator_mock_tests.py
new file mode 100644
index 0000000..96589f9
--- /dev/null
+++ b/tests/unit/replicator_mock_tests.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+# Copyright (C) 2018 IBM Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+_replicator_mock_tests_
+
+replicator module - Mock unit tests for the Replicator class
+"""
+
+import mock
+import unittest
+
+from cloudant.database import CouchDatabase
+from cloudant.replicator import Replicator
+
+from tests.unit.iam_auth_tests import MOCK_API_KEY
+
+
+class ReplicatorDocumentValidationMockTests(unittest.TestCase):
+    """
+    Replicator document validation tests
+    """
+
+    def setUp(self):
+        self.repl_id = 'rep_test'
+
+        self.server_url = 'http://localhost:5984'
+        self.user_ctx = {
+            'name': 'foo',
+            'roles': ['erlanger', 'researcher']
+        }
+
+        self.source_db = 'source_db'
+        self.target_db = 'target_db'
+
+    def setUpClientMocks(self, admin_party=False, iam_api_key=None):
+        m_client = mock.MagicMock()
+        type(m_client).server_url = mock.PropertyMock(
+            return_value=self.server_url)
+
+        type(m_client).admin_party = mock.PropertyMock(
+            return_value=admin_party)
+
+        iam_authenticated = False
+
+        if iam_api_key is not None:
+            iam_authenticated = True
+
+            m_session = mock.MagicMock()
+            type(m_session).get_api_key = mock.PropertyMock(
+                return_value=iam_api_key)
+
+            type(m_client).r_session = mock.PropertyMock(
+                return_value=m_session)
+
+        type(m_client).is_iam_authenticated = mock.PropertyMock(
+            return_value=iam_authenticated)
+
+        return m_client
+
+    def test_using_admin_party_source_and_target(self):
+        m_admin_party_client = self.setUpClientMocks(admin_party=True)
+
+        m_replicator = mock.MagicMock()
+        type(m_replicator).creds = mock.PropertyMock(return_value=None)
+        m_admin_party_client.__getitem__.return_value = m_replicator
+
+        # create source/target databases
+        src = CouchDatabase(m_admin_party_client, self.source_db)
+        tgt = CouchDatabase(m_admin_party_client, self.target_db)
+
+        # trigger replication
+        rep = Replicator(m_admin_party_client)
+        rep.create_replication(src, tgt, repl_id=self.repl_id)
+
+        kcall = m_replicator.create_document.call_args_list
+        self.assertEquals(len(kcall), 1)
+        args, kwargs = kcall[0]
+        self.assertEquals(len(args), 1)
+
+        expected_doc = {
+            '_id': self.repl_id,
+            'source': {'url': '/'.join((self.server_url, self.source_db))},
+            'target': {'url': '/'.join((self.server_url, self.target_db))}
+        }
+
+        self.assertDictEqual(args[0], expected_doc)
+        self.assertTrue(kwargs['throw_on_exists'])
+
+    def test_using_basic_auth_source_and_target(self):
+        test_basic_auth_header = 'abc'
+
+        m_basic_auth_client = self.setUpClientMocks()
+
+        m_replicator = mock.MagicMock()
+        m_basic_auth_client.__getitem__.return_value = m_replicator
+        m_basic_auth_client.basic_auth_str.return_value = test_basic_auth_header
+
+        # create source/target databases
+        src = CouchDatabase(m_basic_auth_client, self.source_db)
+        tgt = CouchDatabase(m_basic_auth_client, self.target_db)
+
+        # trigger replication
+        rep = Replicator(m_basic_auth_client)
+        rep.create_replication(
+            src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx)
+
+        kcall = m_replicator.create_document.call_args_list
+        self.assertEquals(len(kcall), 1)
+        args, kwargs = kcall[0]
+        self.assertEquals(len(args), 1)
+
+        expected_doc = {
+            '_id': self.repl_id,
+            'user_ctx': self.user_ctx,
+            'source': {
+                'headers': {'Authorization': test_basic_auth_header},
+                'url': '/'.join((self.server_url, self.source_db))
+            },
+            'target': {
+                'headers': {'Authorization': test_basic_auth_header},
+                'url': '/'.join((self.server_url, self.target_db))
+            }
+        }
+
+        self.assertDictEqual(args[0], expected_doc)
+        self.assertTrue(kwargs['throw_on_exists'])
+
+    def test_using_iam_auth_source_and_target(self):
+        m_iam_auth_client = self.setUpClientMocks(iam_api_key=MOCK_API_KEY)
+
+        m_replicator = mock.MagicMock()
+        m_iam_auth_client.__getitem__.return_value = m_replicator
+
+        # create source/target databases
+        src = CouchDatabase(m_iam_auth_client, self.source_db)
+        tgt = CouchDatabase(m_iam_auth_client, self.target_db)
+
+        # trigger replication
+        rep = Replicator(m_iam_auth_client)
+        rep.create_replication(
+            src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx)
+
+        kcall = m_replicator.create_document.call_args_list
+        self.assertEquals(len(kcall), 1)
+        args, kwargs = kcall[0]
+        self.assertEquals(len(args), 1)
+
+        expected_doc = {
+            '_id': self.repl_id,
+            'user_ctx': self.user_ctx,
+            'source': {
+                'auth': {'iam': {'api_key': MOCK_API_KEY}},
+                'url': '/'.join((self.server_url, self.source_db))
+            },
+            'target': {
+                'auth': {'iam': {'api_key': MOCK_API_KEY}},
+                'url': '/'.join((self.server_url, self.target_db))
+            }
+        }
+
+        self.assertDictEqual(args[0], expected_doc)
+        self.assertTrue(kwargs['throw_on_exists'])
diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py
index cd79a7a..504500b 100644
--- a/tests/unit/replicator_tests.py
+++ b/tests/unit/replicator_tests.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2015, 2016, 2017 IBM Corp. All rights reserved.
+# Copyright (c) 2015, 2018 IBM Corp. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -161,7 +161,6 @@ class ReplicatorTests(UnitTestDbBase):
         )
         self.replication_ids.append(repl_id['_id'])
 
-    @skip_if_not_cookie_auth
     @flaky(max_runs=3)
     def test_create_replication(self):
         """
@@ -180,7 +179,7 @@ class ReplicatorTests(UnitTestDbBase):
         # Test that the replication document was created
         expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx']
         # If Admin Party mode then user_ctx will not be in the key list
-        if self.client.admin_party:
+        if self.client.admin_party or self.client.is_iam_authenticated:
             expected_keys.pop()
         self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys))
         self.assertEqual(repl_doc['_id'], repl_id)
@@ -238,7 +237,7 @@ class ReplicatorTests(UnitTestDbBase):
         # Test that the replication document was created
         expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx']
         # If Admin Party mode then user_ctx will not be in the key list
-        if self.client.admin_party:
+        if self.client.admin_party or self.client.is_iam_authenticated:
             expected_keys.pop()
         self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys))
         self.assertEqual(repl_doc['_id'], repl_id)
@@ -305,7 +304,6 @@ class ReplicatorTests(UnitTestDbBase):
         match = [repl_id for repl_id in all_repl_ids if repl_id in repl_ids]
         self.assertEqual(set(repl_ids), set(match))
 
-    @skip_if_not_cookie_auth
     def test_retrieve_replication_state(self):
         """
         Test that the replication state can be retrieved for a replication
@@ -347,7 +345,6 @@ class ReplicatorTests(UnitTestDbBase):
             )
             self.assertIsNone(repl_state)
 
-    @skip_if_not_cookie_auth
     def test_stop_replication(self):
         """
         Test that a replication can be stopped.
@@ -359,7 +356,16 @@ class ReplicatorTests(UnitTestDbBase):
             self.target_db,
             repl_id
         )
-        self.replicator.stop_replication(repl_id)
+        max_retry = 3
+        while True:
+            try:
+                max_retry -= 1
+                self.replicator.stop_replication(repl_id)
+                break
+            except requests.HTTPError as err:
+                self.assertEqual(err.response.status_code, 409)
+                if max_retry == 0:
+                    self.fail('Failed to stop replication: {0}'.format(err))
         try:
             # The .fetch() will fail since the replication has been stopped
             # and the replication document has been removed from the db.
@@ -383,7 +389,6 @@ class ReplicatorTests(UnitTestDbBase):
                 'Replication with id {} not found.'.format(repl_id)
             )
 
-    @skip_if_not_cookie_auth
     def test_follow_replication(self):
         """
         Test that follow_replication(...) properly iterates updated