Repository: ansible/tower-cli
Pull request #322 (2017-07-10 19:29:54)

Enable workflow schema update.
Connect #284.

See updates in `WORKFLOWS.md` for behavior changes.


Patch:
diff --git a/docs/WORKFLOWS.md b/docs/WORKFLOWS.md
index 7407b80..292dd0c 100644
--- a/docs/WORKFLOWS.md
+++ b/docs/WORKFLOWS.md
@@ -66,9 +66,9 @@ tower-cli workflow schema workflow1
 
 Here, "workflow1" is the name of the workflow.
 
-### Creating a Schema Definition
+### Creating/updating a Schema Definition
 
-To bulk-create a workflow node network, use the workflow schema command.
+To bulk-create or buld-update a workflow node network, use the workflow schema command.
 The schema is JSON or YAML content, and can be passed in the CLI
 argument, or pointed to a file. The schema is passed as a second positional
 argument, where the first argument references the workflow.
@@ -153,22 +153,27 @@ tower-cli workflow schema workflow2 @schema.yml
 The workflow schema feature populates the workflow node network based on the
 hierarchy structure. Before creating each node, it attempts to find an
 existing node with the specified properties in that location in the
-tree, and will not create a new node if it exists.
+tree, and will not create a new node if it exists. Also, if an existing node
+has no correspondence in the schema, the entire sub-tree based on that node will
+be deleted.
 
-Thus, running the command multiple times should not change the workflow
-structure. To continue with the previous example, subsequent
-invocations of:
+Thus, after running the schema command, the resulting workflow topology will always
+be exactly the same as what is specified in the given schema file. To continue with
+the previous example, subsequent invocations of:
 
 ```bash
 tower-cli workflow schema workflow2 @schema.yml
 tower-cli workflow schema workflow2 @schema.yml
 ```
 
-should not change the network of workflow2.
+should not change the network of workflow2, since `schema.yml` file itself remains
+unchanged. However
 
-The schema command can not delete nodes, so running multiple commands with
-different schemas will result in creating multiple branches and/or
-sub-branches.
+```bash
+tower-cli workflow schema workflow2 @new_schema.yml
+```
+
+will modify topology of workflow2 to exactly the same as what is specified in `new_schema.yml`.
 
 ## Launching Workflow Jobs
 
diff --git a/tower_cli/resources/workflow.py b/tower_cli/resources/workflow.py
index fba2c2e..d81d1ce 100644
--- a/tower_cli/resources/workflow.py
+++ b/tower_cli/resources/workflow.py
@@ -26,6 +26,130 @@ import click
 from collections import deque
 
 
+class TreeNode(object):
+    def __init__(self, data, wfjt, include_id=False):
+        ujt_attrs = list(JOB_TYPES.values())
+        FK_FIELDS = ujt_attrs + ['inventory', 'credential']
+        node_attrs = {}
+        attr_names = NODE_STANDARD_FIELDS + ujt_attrs
+        if include_id:
+            attr_names.append('id')
+        for fd in attr_names:
+            if fd not in data:
+                continue
+            if fd in FK_FIELDS and not isinstance(data[fd], int):
+                # Node's template was given by name, do lookup
+                ujt_res = get_resource(fd)
+                ujt_data = ujt_res.get(name=data[fd])
+                node_attrs[fd] = ujt_data['id']
+            else:
+                node_attrs[fd] = data[fd]
+        node_attrs['workflow_job_template'] = wfjt
+        for ujt_name in ujt_attrs:
+            if ujt_name not in node_attrs:
+                continue
+            if 'unified_job_template' not in node_attrs:
+                node_attrs['unified_job_template'] = node_attrs.pop(ujt_name)
+            else:
+                raise BadRequest(
+                    'You should not provide more than one of the attributes'
+                    ' job_template, project and inventory_source.'
+                )
+        self.unified_job_template = node_attrs.get('unified_job_template', None)
+        self.node_attrs = node_attrs
+        for rel in ['success_nodes', 'failure_nodes', 'always_nodes']:
+            setattr(
+                self, rel,
+                [TreeNode(x, wfjt, include_id=include_id) for x in data.get(rel, data.get(rel[: -6], []))]
+            )
+
+    def create(self, node_res):
+        self.node_attrs['id'] = node_res.create(**self.node_attrs)['id']
+        queue = deque()
+        queue.append(self)
+        while queue:
+            node = queue.popleft()
+            for rel in ['success_nodes', 'failure_nodes', 'always_nodes']:
+                for sub_node in getattr(node, rel, []):
+                    sub_node.node_attrs['id'] = node_res.create(**sub_node.node_attrs)['id']
+                    getattr(node_res, 'associate_%s' % rel[:-1])(
+                        node.node_attrs['id'], child=sub_node.node_attrs['id']
+                    )
+                    queue.append(sub_node)
+
+    def delete(self, node_res):
+        for rel in ['success_nodes', 'failure_nodes', 'always_nodes']:
+            for sub_node in getattr(self, rel, []):
+                sub_node.delete(node_res)
+        node_res.delete(pk=self.node_attrs['id'])
+
+
+def _compare_node_lists(old, new):
+    '''
+    Investigate two lists of workflow TreeNodes and categorize them.
+
+    There will be three types of nodes after categorization:
+        1. Nodes that only exists in the new list. These nodes will later be
+        created recursively.
+        2. Nodes that only exists in the old list. These nodes will later be
+        deleted recursively.
+        3. Node pairs that makes an exact match. These nodes will be further
+        investigated.
+
+    Corresponding nodes of old and new lists will be distinguished by their
+    unified_job_template value. A special case is that both the old and the new
+    lists contain one type of node, say A, and at least one of them contains
+    duplicates. In this case all A nodes in the old list will be categorized as
+    to-be-deleted and all A nodes in the new list will be categorized as
+    to-be-created.
+    '''
+    to_expand = []
+    to_delete = []
+    to_recurse = []
+    old_records = {}
+    new_records = {}
+    for tree_node in old:
+        old_records.setdefault(tree_node.unified_job_template, [])
+        old_records[tree_node.unified_job_template].append(tree_node)
+    for tree_node in new:
+        new_records.setdefault(tree_node.unified_job_template, [])
+        new_records[tree_node.unified_job_template].append(tree_node)
+    for ujt_id in old_records:
+        if ujt_id not in new_records:
+            to_delete.extend(old_records[ujt_id])
+            continue
+        old_list = old_records[ujt_id]
+        new_list = new_records.pop(ujt_id)
+        if len(old_list) == 1 and len(new_list) == 1:
+            to_recurse.append((old_list[0], new_list[0]))
+        else:
+            to_delete.extend(old_list)
+            to_expand.extend(new_list)
+    for nodes in new_records.values():
+        to_expand.extend(nodes)
+    return to_expand, to_delete, to_recurse
+
+
+def _do_update_workflow(existing_roots, updated_roots, node_res):
+    to_expand, to_delete, to_recurse = _compare_node_lists(existing_roots, updated_roots)
+    for node in to_delete:
+        node.delete(node_res)
+    for node in to_expand:
+        node.create(node_res)
+    for old_node, new_node in to_recurse:
+        for rel in ['success_nodes', 'failure_nodes', 'always_nodes']:
+            to_assoc = _do_update_workflow(getattr(old_node, rel, []), getattr(new_node, rel, []), node_res)
+            for sub_node in to_assoc:
+                getattr(node_res, 'associate_%s' % rel[:-1])(old_node.node_attrs['id'], child=sub_node.node_attrs['id'])
+    return to_expand
+
+
+def _update_workflow(existing_roots, updated_roots):
+    # Node resource should be fetched *only once*.
+    node_res = get_resource('node')
+    _do_update_workflow(existing_roots, updated_roots, node_res)
+
+
 class Resource(models.SurveyResource):
     cli_help = 'Manage workflow job templates.'
     endpoint = '/workflow_job_templates/'
@@ -85,7 +209,7 @@ class Resource(models.SurveyResource):
         def branch_schema(node_id):
             i = node_list_pos[node_id]
             node_dict = node_results[i]
-            ret_dict = {}
+            ret_dict = {"id": node_id}
             for fd in NODE_STANDARD_FIELDS:
                 val = node_dict.get(fd, None)
                 if val is not None:
@@ -131,71 +255,23 @@ class Resource(models.SurveyResource):
         node_network param is given.
         If not, print a YAML representation of the node network.
         """
+        existing_network = self._get_schema(wfjt)
+        if not isinstance(existing_network, list):
+            existing_network = []
         if node_network is None:
             if settings.format == 'human':
                 settings.format = 'yaml'
-            return self._get_schema(wfjt)
-
-        node_res = get_resource('node')
-
-        def create_node(node_branch, parent, relationship):
-            # Create node with data specified by top-level keys
-            create_data = {}
-            FK_FIELDS = JOB_TYPES.values() + ['inventory', 'credential']
-            for fd in NODE_STANDARD_FIELDS + JOB_TYPES.values():
-                if fd in node_branch:
-                    if (
-                            fd in FK_FIELDS and
-                            not isinstance(node_branch[fd], int)):
-                        # Node's template was given by name, do lookup
-                        ujt_res = get_resource(fd)
-                        ujt_data = ujt_res.get(name=node_branch[fd])
-                        create_data[fd] = ujt_data['id']
-                    else:
-                        create_data[fd] = node_branch[fd]
-            create_data['workflow_job_template'] = wfjt
-            return node_res._get_or_create_child(
-                parent, relationship, **create_data)
-
-        def get_adj_list(node_branch):
-            ret = {}
-            for fd in node_branch:
-                for rel in ['success', 'failure', 'always']:
-                    if fd.startswith(rel):
-                        sub_branch_list = node_branch[fd]
-                        if not isinstance(sub_branch_list, list):
-                            raise BadRequest(
-                                'Sublists in spec must be lists.'
-                                'Encountered in {0} at {1}'.format(
-                                    fd, sub_branch_list))
-                        ret[rel] = sub_branch_list
-                        break
-            return ret
-
-        def create_node_recursive(node_network):
-            queue = deque()
-            id_queue = deque()
-            for base_node in node_network:
-                queue.append(base_node)
-                id_queue.append(create_node(base_node, None, None)['id'])
-                while (len(queue) != 0):
-                    to_expand = queue.popleft()
-                    parent_id = id_queue.popleft()
-                    adj_list = get_adj_list(to_expand)
-                    for rel in adj_list:
-                        for sub_node in adj_list[rel]:
-                            id_queue.append(create_node(sub_node, parent_id,
-                                                        rel)['id'])
-                            queue.append(sub_node)
-                            node_res._assoc(node_res._forward_rel_name(rel),
-                                            parent_id, id_queue[-1])
+            return existing_network
 
         if hasattr(node_network, 'read'):
             node_network = node_network.read()
         node_network = string_to_dict(
             node_network, allow_kv=False, require_dict=False)
+        if not isinstance(node_network, list):
+            node_network = []
 
-        create_node_recursive(node_network)
+        _update_workflow([TreeNode(x, wfjt, include_id=True) for x in existing_network],
+                         [TreeNode(x, wfjt) for x in node_network])
 
         if settings.format == 'human':
             settings.format = 'yaml'


Test patch:
diff --git a/tests/test_resources_workflow.py b/tests/test_resources_workflow.py
index 0ba4e32..d8dc00b 100644
--- a/tests/test_resources_workflow.py
+++ b/tests/test_resources_workflow.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+
 import tower_cli
 from tower_cli.resources.workflow import Resource
 
@@ -47,10 +49,344 @@ EXAMPLE_NODE_LIST = [
 ]
 
 
+CUR_WORKFLOW = [
+    {
+        'id': 1,
+        'success_nodes': [],
+        'failure_nodes': [],
+        'always_nodes': [2, 3],
+        'workflow_job_template': 1,
+        'unified_job_template': 1,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'project_update'
+            }
+        }
+    },
+    {
+        'id': 2,
+        'success_nodes': [4],
+        'failure_nodes': [5],
+        'always_nodes': [],
+        'workflow_job_template': 1,
+        'unified_job_template': 2,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'project_update'
+            }
+        }
+    },
+    {
+        'id': 3,
+        'success_nodes': [],
+        'failure_nodes': [6, 7],
+        'always_nodes': [],
+        'workflow_job_template': 1,
+        'unified_job_template': 3,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'project_update'
+            }
+        }
+    },
+    {
+        'id': 4,
+        'success_nodes': [],
+        'failure_nodes': [],
+        'always_nodes': [],
+        'workflow_job_template': 1,
+        'unified_job_template': 4,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'job'
+            }
+        }
+    },
+    {
+        'id': 5,
+        'success_nodes': [],
+        'failure_nodes': [],
+        'always_nodes': [],
+        'workflow_job_template': 1,
+        'unified_job_template': 5,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'job'
+            }
+        }
+    },
+    {
+        'id': 6,
+        'success_nodes': [],
+        'failure_nodes': [],
+        'always_nodes': [],
+        'workflow_job_template': 1,
+        'unified_job_template': 6,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'job'
+            }
+        }
+    },
+    {
+        'id': 7,
+        'success_nodes': [],
+        'failure_nodes': [],
+        'always_nodes': [],
+        'workflow_job_template': 1,
+        'unified_job_template': 7,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'job'
+            }
+        }
+    },
+    {
+        'id': 8,
+        'success_nodes': [9, 10],
+        'failure_nodes': [],
+        'always_nodes': [],
+        'workflow_job_template': 1,
+        'unified_job_template': 8,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'inventory_update'
+            }
+        }
+    },
+    {
+        'id': 9,
+        'success_nodes': [],
+        'failure_nodes': [],
+        'always_nodes': [11],
+        'workflow_job_template': 1,
+        'unified_job_template': 9,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'inventory_update'
+            }
+        }
+    },
+    {
+        'id': 10,
+        'success_nodes': [],
+        'failure_nodes': [],
+        'always_nodes': [12],
+        'workflow_job_template': 1,
+        'unified_job_template': 9,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'inventory_update'
+            }
+        }
+    },
+    {
+        'id': 11,
+        'success_nodes': [],
+        'failure_nodes': [],
+        'always_nodes': [],
+        'workflow_job_template': 1,
+        'unified_job_template': 10,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'job'
+            }
+        }
+    },
+    {
+        'id': 12,
+        'success_nodes': [],
+        'failure_nodes': [],
+        'always_nodes': [],
+        'workflow_job_template': 1,
+        'unified_job_template': 10,
+        'summary_fields': {
+            'unified_job_template': {
+                'unified_job_type': 'job'
+            }
+        }
+    },
+]
+
+
+UPDATED_WORKFLOW = [
+    {
+        u'project': 1,
+        u'always': [
+            {
+                u'project': 2,
+                u'success': [
+                    {
+                        u'job_template': 4
+                    }
+                ],
+                u'failure': [
+                    {
+                        u'job_template': 5,
+                        u'success': [
+                            {
+                                u'job_template': 6
+                            },
+                            {
+                                u'job_template': 7
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    },
+    {
+        u'inventory_source': 8,
+        u'success': [
+            {
+                u'inventory_source': 9,
+                u'always': [
+                    {
+                        u'job_template': 10,
+                    }
+                ]
+            },
+            {
+                u'inventory_source': 9,
+                u'always': [
+                    {
+                        u'job_template': 10,
+                    }
+                ]
+            }
+        ]
+    }
+]
+
+
+UPDATED_RESULT = [
+    {
+        u'project': 1,
+        u'always_nodes': [
+            {
+                u'project': 2,
+                u'success_nodes': [
+                    {
+                        u'job_template': 4
+                    }
+                ],
+                u'failure_nodes': [
+                    {
+                        u'job_template': 5,
+                        u'success_nodes': [
+                            {
+                                u'job_template': 6
+                            },
+                            {
+                                u'job_template': 7
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    },
+    {
+        u'inventory_source': 8,
+        u'success_nodes': [
+            {
+                u'job_template': 9,
+                u'always_nodes': [
+                    {
+                        u'job_template': 10,
+                    }
+                ]
+            },
+            {
+                u'job_template': 9,
+                u'always_nodes': [
+                    {
+                        u'job_template': 10,
+                    }
+                ]
+            }
+        ]
+    }
+]
+
+
+class NodeResMock(object):
+    def __init__(self, init_list):
+        self.db = {}
+        self.cur_pk = 0
+        for node in init_list:
+            self.db[node['id']] = node
+            self.cur_pk = max(self.cur_pk, node['id'])
+        self.create_cnt = 0
+        self.delete_cnt = 0
+        self.associate_success_cnt = 0
+        self.associate_failure_cnt = 0
+        self.associate_always_cnt = 0
+
+    def create(self, **kwargs):
+        self.cur_pk += 1
+        new_record = {
+            'success_nodes': [],
+            'failure_nodes': [],
+            'always_nodes': [],
+            'id': self.cur_pk,
+            'summary_fields': {
+                'unified_job_template': {
+                    'unified_job_type': 'job'
+                }
+            }
+        }
+        new_record.update(kwargs)
+        self.db[new_record['id']] = new_record
+        self.create_cnt += 1
+        return new_record
+
+    def list(self, **kwargs):
+        return {
+            'count': len(self.db),
+            'results': list(self.db.values()),
+            'next': None,
+            'previous': None
+        }
+
+    def delete(self, pk=None):
+        if not pk:
+            return
+        self.db.pop(pk, None)
+        for record in self.db.values():
+            for rel in ['success_nodes', 'failure_nodes', 'always_nodes']:
+                try:
+                    record[rel].remove(pk)
+                except ValueError:
+                    pass
+        self.delete_cnt += 1
+
+    def associate_success_node(self, id, child=None):
+        if not child or id not in self.db:
+            return
+        self.db[id]['success_nodes'].append(child)
+        self.associate_success_cnt += 1
+
+    def associate_failure_node(self, id, child=None):
+        if not child or id not in self.db:
+            return
+        self.db[id]['failure_nodes'].append(child)
+        self.associate_failure_cnt += 1
+
+    def associate_always_node(self, id, child=None):
+        if not child or id not in self.db:
+            return
+        self.db[id]['always_nodes'].append(child)
+        self.associate_always_cnt += 1
+
+
 class SchemaTests(unittest.TestCase):
     """
     Tests for the schema file in the tower_cli utils
     """
+    def setUp(self):
+        self.res = tower_cli.get_resource('workflow')
 
     def test_expected_output(self):
         """
@@ -63,8 +399,10 @@ class SchemaTests(unittest.TestCase):
             [
                 {
                     'job_template': 48,
+                    'id': 1,
                     'always_nodes': [
                         {
+                            'id': 2,
                             'project': 98
                         }
                     ]
@@ -72,6 +410,25 @@ class SchemaTests(unittest.TestCase):
             ]
         )
 
+    def _remove_ids(self, input_list):
+        for node in input_list:
+            node.pop('id', None)
+            for rel in ['success_nodes', 'failure_nodes', 'always_nodes']:
+                if rel in node:
+                    self._remove_ids(node[rel])
+        return input_list
+
+    def test_schema_update(self):
+        node_res_mock = NodeResMock(CUR_WORKFLOW)
+        with mock.patch('tower_cli.resources.workflow.get_resource', return_value=node_res_mock):
+            res = self._remove_ids(self.res.schema(1, node_network=json.dumps(UPDATED_WORKFLOW)))
+            self.assertEqual(res, UPDATED_RESULT)
+            self.assertEqual(node_res_mock.create_cnt, 6)
+            self.assertEqual(node_res_mock.delete_cnt, 7)
+            self.assertEqual(node_res_mock.associate_success_cnt, 4)
+            self.assertEqual(node_res_mock.associate_failure_cnt, 0)
+            self.assertEqual(node_res_mock.associate_always_cnt, 2)
+
 
 class NodeModelTests(unittest.TestCase):
     """