Repository: ansible/tower-cli
Pull request #741 (2019-10-24 15:01:40)

Get send/receive to work with AWX 8.0.0 and JT credentials
Closes https://github.com/ansible/tower-cli/issues/716

Yanked the JT code from https://github.com/ansible/tower-cli/pull/610

This is so that the `tower_send` and `tower_receive` modules become self-consistent (creates the data it receives) for the AWX collection version 8.0.0+


Patch:
diff --git a/README.rst b/README.rst
index 655c723..feda9d6 100644
--- a/README.rst
+++ b/README.rst
@@ -10,6 +10,10 @@ It is also what the Ansible `tower_*` modules use under the hood. Such as:
 
 https://docs.ansible.com/ansible/latest/modules/tower_organization_module.html
 
+These modules are now vendored as part of the AWX collection at:
+
+https://galaxy.ansible.com/awx/awx
+
 Supporting correct operation of the modules is the maintenance aim of this
 package.
 
diff --git a/docs/source/HISTORY.rst b/docs/source/HISTORY.rst
index 6e51b43..3e0725e 100644
--- a/docs/source/HISTORY.rst
+++ b/docs/source/HISTORY.rst
@@ -1,6 +1,12 @@
 Release History
 ===============
 
+3.3.7 (2019-10-25)
+------------------
+
+- Job template associate_credential now uses "credentials" endpoint
+- Include related credentials to job templates in send and receive commands
+
 3.3.6 (2019-07-19)
 ------------------
 
diff --git a/tower_cli/cli/transfer/common.py b/tower_cli/cli/transfer/common.py
index d651b9f..acff3d2 100644
--- a/tower_cli/cli/transfer/common.py
+++ b/tower_cli/cli/transfer/common.py
@@ -96,6 +96,9 @@ def map_node_to_post_options(post_options, source_node, target_node):
         if 'required' in post_options[option] and post_options[option]["required"]:
             target_node[option] = source_node[option]
         elif option in source_node and source_node[option] != default:
+            # work-around AWX bug in 8.0.0 where webhook_service OPTIONS not right
+            if option == 'webhook_service' and source_node[option] == '':
+                continue
             target_node[option] = source_node[option]
 
 
@@ -419,12 +422,12 @@ def get_assets_from_input(all=False, asset_input=None):
     return return_assets
 
 
-def extract_extra_credentials(asset):
+def extract_credentials(asset):
     return_credentials = []
     name_to_id_map = {}
 
-    extra_credentials = load_all_assets(asset['related']['extra_credentials'])
-    for a_credential in extra_credentials['results']:
+    credentials = load_all_assets(asset['related']['credentials'])
+    for a_credential in credentials['results']:
         name_to_id_map[a_credential['name']] = a_credential['id']
         return_credentials.append(a_credential['name'])
 
diff --git a/tower_cli/cli/transfer/receive.py b/tower_cli/cli/transfer/receive.py
index 6a4bfd0..0a91807 100644
--- a/tower_cli/cli/transfer/receive.py
+++ b/tower_cli/cli/transfer/receive.py
@@ -97,9 +97,9 @@ class Receiver:
                             exported_asset[common.ASSET_RELATION_KEY][notification_type] = \
                                 common.extract_notifications(asset, notification_type)
 
-                    elif relation == 'extra_credentials':
+                    elif relation == 'credentials':
                         exported_asset[common.ASSET_RELATION_KEY][relation] =\
-                            common.extract_extra_credentials(asset)['items']
+                            common.extract_credentials(asset)['items']
 
                     elif relation == 'schedules':
                         exported_asset[common.ASSET_RELATION_KEY][relation] =\
diff --git a/tower_cli/cli/transfer/send.py b/tower_cli/cli/transfer/send.py
index 6547021..3696f2f 100644
--- a/tower_cli/cli/transfer/send.py
+++ b/tower_cli/cli/transfer/send.py
@@ -1,5 +1,7 @@
 import tower_cli
 import json
+import six
+import re
 from tower_cli.exceptions import TowerCLIError, CannotStartJob, JobFailure
 import tower_cli.cli.transfer.common as common
 from tower_cli.cli.transfer.logging_command import LoggingCommand
@@ -96,7 +98,7 @@ class Sender(LoggingCommand):
                                 # First use the API to get the user
                                 from tower_cli.api import Client
                                 api_client = Client()
-                                me_response = api_client.request('GET', 'me')
+                                me_response = api_client.request('GET', 'me/')
                                 response_json = me_response.json()
                                 if 'results' not in response_json or 'id' not in response_json['results'][0]:
                                     raise TowerCLIError("Unable to get user information from Tower")
@@ -287,8 +289,8 @@ class Sender(LoggingCommand):
                             self.import_inventory_groups(existing_object, relations[a_relation])
                         elif a_relation in common.NOTIFICATION_TYPES:
                             self.import_notification_relations(existing_object, relations[a_relation], a_relation)
-                        elif a_relation == 'extra_credentials':
-                            self.import_extra_credentials(existing_object, relations[a_relation])
+                        elif a_relation == 'credentials':
+                            self.import_credentials(existing_object, relations[a_relation])
                         elif a_relation == 'schedules':
                             schedules_to_import.append(relations[a_relation])
                         elif a_relation == 'roles':
@@ -396,20 +398,33 @@ class Sender(LoggingCommand):
                 # Choices is an array like [ [ value, label ], [value, label], ... ]
                 valid_options = []
                 for a_choice in post_options[option]["choices"]:
-                    value = a_choice[0]
-                    label = a_choice[1]
+                    if isinstance(a_choice, six.string_types):
+                        m = re.match(r"^\('(?P<value>.*)',\s'(?P<label>.*)'\)$", a_choice)
+                        if m:
+                            value = m.group('value')
+                            label = m.group('label')
+                        else:
+                            raise Exception('no!')
+                            value = a_choice
+                            label = None
+                    else:
+                        value = a_choice[0]
+                        label = a_choice[1]
                     valid_options.append(value)
-                    valid_options.append(label)
+                    if label is not None:
+                        valid_options.append(label)
 
-                    if an_asset[option] == label:
+                    if label is not None and an_asset[option] == label:
                         an_asset[option] = value
                         valid_choice = True
                     elif an_asset[option] == value:
                         valid_choice = True
 
+                # if 'git' in valid_options:
+                #     raise Exception('valid choices: {}'.format(valid_options))
                 if not valid_choice:
-                    self.log_error("Value {} is not a valid choice for option {} for {} {}".format(
-                        an_asset[option], option, asset_type, name
+                    self.log_error("Value {} is not a valid choice for option {} for {} {}. Options: {}".format(
+                        an_asset[option], option, asset_type, name, valid_options
                     ))
                     post_check_succeeded = False
 
@@ -490,14 +505,14 @@ class Sender(LoggingCommand):
                     # Someday we could go and try to resolve inventory projects or scripts
                     pass
 
-                elif relation == 'extra_credentials':
+                elif relation == 'credentials':
                     for credential in an_asset[common.ASSET_RELATION_KEY][relation]:
                         if 'credential' in self.sorted_assets and credential in self.sorted_assets['credential']:
                             continue
                         try:
                             tower_cli.get_resource('credential').get(**{'name': credential})
                         except TowerCLIError:
-                            self.log_error("Unable to resolve extra_credential {}".format(credential))
+                            self.log_error("Unable to resolve credential {}".format(credential))
                             post_check_succeeded = False
 
                 elif relation == 'schedules':
@@ -926,10 +941,10 @@ class Sender(LoggingCommand):
             del a_node['unified_job_type']
             del a_node['unified_job_name']
 
-    def import_extra_credentials(self, existing_object, new_creds):
+    def import_credentials(self, existing_object, new_creds):
         # Credentials are just an array of names
         # So importing these can be done very easily by comparing new_creds vs existing_creds
-        existing_creds_data = common.extract_extra_credentials(existing_object)
+        existing_creds_data = common.extract_credentials(existing_object)
         existing_creds = existing_creds_data['items']
         existing_name_to_id = existing_creds_data['existing_name_to_id_map']
         if existing_creds == new_creds:
@@ -942,23 +957,23 @@ class Sender(LoggingCommand):
                 tower_cli.get_resource('job_template').disassociate_credential(
                     existing_object['id'], existing_name_to_id[cred]
                 )
-                self.log_change("Removed extra credential {}".format(cred))
+                self.log_change("Removed credential {}".format(cred))
             except TowerCLIError as e:
-                self.log_error("Unable to remove extra credential {} : {}".format(cred, e))
+                self.log_error("Unable to remove credential {} : {}".format(cred, e))
 
         # Creds to add is the difference between existing_creds and extra_creds
         for cred in list(set(new_creds).difference(existing_creds)):
             try:
                 new_credential = tower_cli.get_resource('credential').get(**{'name': cred})
             except TowerCLIError as e:
-                self.log_error("Unable to resolve extra credential {} : {}".format(cred, e))
+                self.log_error("Unable to resolve credential {} : {}".format(cred, e))
                 continue
 
             try:
                 tower_cli.get_resource('job_template').associate_credential(existing_object['id'], new_credential['id'])
-                self.log_change("Added extra credential {}".format(cred))
+                self.log_change("Added credential {}".format(cred))
             except TowerCLIError as e:
-                self.log_error("Unable to add extra credential {} : ".format(cred, e))
+                self.log_error("Unable to add credential {} : ".format(cred, e))
 
     def import_labels(self, existing_object, new_labels, asset_type):
         existing_labels_data = common.extract_labels(existing_object)
diff --git a/tower_cli/constants.py b/tower_cli/constants.py
index b642578..b2db694 100644
--- a/tower_cli/constants.py
+++ b/tower_cli/constants.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 
-VERSION = '3.3.6'
+VERSION = '3.3.7'
 # This is the release number for the RPM builds
 RELEASE = 1
 CUR_API_VERSION = 'v2'
diff --git a/tower_cli/resources/job_template.py b/tower_cli/resources/job_template.py
index 4af50e0..dc03ff7 100644
--- a/tower_cli/resources/job_template.py
+++ b/tower_cli/resources/job_template.py
@@ -18,9 +18,10 @@ from __future__ import absolute_import, unicode_literals
 import click
 
 from tower_cli import models, resources
-from tower_cli.utils import parser
+from tower_cli.utils import parser, debug
 from tower_cli.api import client
 from tower_cli.cli import types
+from tower_cli.exceptions import NotFound
 
 
 class Resource(models.SurveyResource):
@@ -28,7 +29,7 @@ class Resource(models.SurveyResource):
     cli_help = 'Manage job templates.'
     endpoint = '/job_templates/'
     dependencies = ['inventory', 'credential', 'project', 'vault_credential']
-    related = ['survey_spec', 'notification_templates', 'extra_credentials', 'schedules', 'labels']
+    related = ['survey_spec', 'notification_templates', 'schedules', 'labels', 'credentials']
 
     name = models.Field(unique=True)
     description = models.Field(required=False, display=False)
@@ -118,12 +119,22 @@ class Resource(models.SurveyResource):
     labels = models.ManyToManyField('label')
     instance_groups = models.ManyToManyField('instance_group', method_name='ig')
 
-    def write(self, *args, **kwargs):
+    def write(self, pk=None, *args, **kwargs):
         # Provide a default value for job_type, but only in creation of JT
         if (kwargs.get('create_on_missing', False) and
                 (not kwargs.get('job_type', None))):
             kwargs['job_type'] = 'run'
-        return super(Resource, self).write(*args, **kwargs)
+        mcred = kwargs.get('credential', None)
+        ret = super(Resource, self).write(pk=pk, **kwargs)
+        cred_ids = [c['id'] for c in ret.get('summary_fields', {}).get('credentials', [])]
+        if mcred and mcred not in cred_ids:
+            new_pk = ret['id']
+            debug.log('Processing deprecated credential field via another request.', header='details')
+            self._assoc('credentials', new_pk, mcred)
+            ret = self.read(new_pk)
+            ret['id'] = new_pk
+            ret['changed'] = True
+        return ret
 
     @resources.command(use_fields_as_options=False)
     @click.option('--job-template', type=types.Related('job_template'))
@@ -143,7 +154,13 @@ class Resource(models.SurveyResource):
 
         =====API DOCS=====
         """
-        return self._assoc('extra_credentials', job_template, credential)
+        try:
+            # Tower 3.3 behavior, allows all types of credentials
+            return self._assoc('credentials', job_template, credential)
+        except NotFound:
+            debug.log('Attempting to use extra_credential as fallback in '
+                      'case server is older version.', header='details')
+            return self._assoc('extra_credentials', job_template, credential)
 
     @resources.command(use_fields_as_options=False)
     @click.option('--job-template', type=types.Related('job_template'))
@@ -163,7 +180,12 @@ class Resource(models.SurveyResource):
 
         =====API DOCS=====
         """
-        return self._disassoc('extra_credentials', job_template, credential)
+        try:
+            return self._disassoc('credentials', job_template, credential)
+        except NotFound:
+            debug.log('Attempting to use extra_credential as fallback in '
+                      'case server is older version.', header='details')
+            return self._disassoc('extra_credentials', job_template, credential)
 
     @resources.command(use_fields_as_options=False)
     @click.option('--job-template', type=types.Related('job_template'))


Test patch:
diff --git a/tests/test_resources_job_template.py b/tests/test_resources_job_template.py
index b5e088d..612e5a3 100644
--- a/tests/test_resources_job_template.py
+++ b/tests/test_resources_job_template.py
@@ -41,7 +41,7 @@ class TemplateTests(unittest.TestCase):
             t.register_json(endpoint, {'changed': True, 'id': 42},
                             method='POST')
             self.res.create(name='bar', job_type='run', inventory=1,
-                            project=1, playbook='foobar.yml', credential=1)
+                            project=1, playbook='foobar.yml')
             self.assertEqual(t.requests[0].method, 'GET')
             self.assertEqual(t.requests[1].method, 'POST')
             self.assertEqual(len(t.requests), 2)
@@ -55,7 +55,7 @@ class TemplateTests(unittest.TestCase):
             t.register_json(endpoint, {'changed': True, 'id': 42},
                             method='POST')
             self.res.create(name='bar', inventory=1, project=1,
-                            playbook='foobar.yml', credential=1)
+                            playbook='foobar.yml')
             req_body = json.loads(t.requests[1].body)
             self.assertIn('job_type', req_body)
             self.assertEqual(req_body['job_type'], 'run')
@@ -74,13 +74,13 @@ class TemplateTests(unittest.TestCase):
                                 'playbook': 'foobar.yml', 'credential': 1},
                             method='POST')
             self.res.create(name='bar', job_type='run', inventory=1,
-                            project=1, playbook='foobar.yml', credential=1)
+                            project=1, playbook='foobar.yml')
 
             f = ResSubcommand(self.res)._echo_method(self.res.create)
             with mock.patch.object(click, 'secho'):
                 with settings.runtime_values(format='human'):
                     f(name='bar', job_type='run', inventory=1,
-                      project=1, playbook='foobar.yml', credential=1)
+                      project=1, playbook='foobar.yml')
 
     def test_create_w_extra_vars(self):
         """Establish that a job template can be created
@@ -94,7 +94,7 @@ class TemplateTests(unittest.TestCase):
             t.register_json(endpoint, {'changed': True, 'id': 42},
                             method='POST')
             self.res.create(name='bar', job_type='run', inventory=1,
-                            project=1, playbook='foobar.yml', credential=1,
+                            project=1, playbook='foobar.yml',
                             extra_vars=['foo: bar'])
             self.assertEqual(t.requests[0].method, 'GET')
             self.assertEqual(t.requests[1].method, 'POST')
@@ -137,9 +137,9 @@ class TemplateTests(unittest.TestCase):
         that we expect.
         """
         with client.test_mode as t:
-            t.register_json('/job_templates/42/extra_credentials/?id=84',
+            t.register_json('/job_templates/42/credentials/?id=84',
                             {'count': 0, 'results': []})
-            t.register_json('/job_templates/42/extra_credentials/', {}, method='POST')
+            t.register_json('/job_templates/42/credentials/', {}, method='POST')
             self.res.associate_credential(42, 84)
             self.assertEqual(t.requests[1].body,
                              json.dumps({'associate': True, 'id': 84}))
@@ -149,10 +149,10 @@ class TemplateTests(unittest.TestCase):
         that we expect.
         """
         with client.test_mode as t:
-            t.register_json('/job_templates/42/extra_credentials/?id=84',
+            t.register_json('/job_templates/42/credentials/?id=84',
                             {'count': 1, 'results': [{'id': 84}],
                              'next': None, 'previous': None})
-            t.register_json('/job_templates/42/extra_credentials/', {}, method='POST')
+            t.register_json('/job_templates/42/credentials/', {}, method='POST')
             self.res.disassociate_credential(42, 84)
             self.assertEqual(t.requests[1].body,
                              json.dumps({'disassociate': True, 'id': 84}))