Merge pull request #9895 from sean-m-sullivan/workflow_schema

Tower workflow schema

SUMMARY
See #9309 This is a clean PR of that, after an errant rebase
Adds a way to add entire workflow node schemas to workflows. Either through the workflow schema module or the workflow job template module.
This speeds up workflow creation vs the workflow node module by 3x.
The model for the schemas is the format used by the tower_export module.
The main difference between this and the workflow node module is that the loops are done in python. Traditionally if you have a workflow with 10 nodes, ansible tasks need to be invoked 19 times. 1x to create the workflow, 10 x to initially create the nodes, and then one time for each node that is not an endpoint in the schema. This removes the need to loop and invoke many times.
ISSUE TYPE
Feature Pull Request

COMPONENT NAME
awx-collection
AWX VERSION
17.0.1

Reviewed-by: John Westcott IV <None>
Reviewed-by: Bianca Henderson <beeankha@gmail.com>

awx_collection/plugins/module_utils/tower_api.py

@@ -474,7 +474,7 @@ class TowerAPIModule(TowerModule):
         #    1. None if the existing_item is already defined (so no create needs to happen)
         #    2. The response from Tower from calling the patch on the endpont. It's up to you to process the response and exit from the module
         # Note: common error codes from the Tower API can cause the module to fail
-
+        response = None
         if not endpoint:
             self.fail_json(msg="Unable to create new {0} due to missing endpoint".format(item_type))
 
@@ -522,8 +522,11 @@ class TowerAPIModule(TowerModule):
         elif auto_exit:
             self.exit_json(**self.json_output)
         else:
-            last_data = response['json']
-            return last_data
+            if response is not None:
+                last_data = response['json']
+                return last_data
+            else:
+                return
 
     def _encrypted_changed_warning(self, field, old, warning=False):
         if not warning:

awx_collection/plugins/modules/tower_workflow_job_template.py

@@ -20,7 +20,7 @@ short_description: create, update, or destroy Ansible Tower workflow job templat
 description:
     - Create, update, or destroy Ansible Tower workflow job templates.
     - Replaces the deprecated tower_workflow_template module.
-    - Use the tower_workflow_job_template_node after this to build the workflow's graph.
+    - Use the tower_workflow_job_template_node after this, or use the schema paramater to build the workflow's graph
 options:
     name:
       description:
@@ -144,6 +144,185 @@ options:
         - list of notifications to send on start
       type: list
       elements: str
+    schema:
+      description:
+        - A json list of nodes and their coresponding options. The following suboptions describe a single node.
+      type: list
+      suboptions:
+        extra_data:
+          description:
+            - Variables to apply at launch time.
+            - Will only be accepted if job template prompts for vars or has a survey asking for those vars.
+          type: dict
+          default: {}
+        inventory:
+          description:
+            - Inventory applied as a prompt, if job template prompts for inventory
+          type: str
+        scm_branch:
+          description:
+            - SCM branch applied as a prompt, if job template prompts for SCM branch
+          type: str
+        job_type:
+          description:
+            - Job type applied as a prompt, if job template prompts for job type
+          type: str
+          choices:
+            - 'run'
+            - 'check'
+        job_tags:
+          description:
+            - Job tags applied as a prompt, if job template prompts for job tags
+          type: str
+        skip_tags:
+          description:
+            - Tags to skip, applied as a prompt, if job tempalte prompts for job tags
+          type: str
+        limit:
+          description:
+            - Limit to act on, applied as a prompt, if job template prompts for limit
+          type: str
+        diff_mode:
+          description:
+            - Run diff mode, applied as a prompt, if job template prompts for diff mode
+          type: bool
+        verbosity:
+          description:
+            - Verbosity applied as a prompt, if job template prompts for verbosity
+          type: str
+          choices:
+            - '0'
+            - '1'
+            - '2'
+            - '3'
+            - '4'
+            - '5'
+        all_parents_must_converge:
+          description:
+            - If enabled then the node will only run if all of the parent nodes have met the criteria to reach this node
+          type: bool
+        identifier:
+          description:
+            - An identifier for this node that is unique within its workflow.
+            - It is copied to workflow job nodes corresponding to this node.
+          required: True
+          type: str
+        state:
+          description:
+            - Desired state of the resource.
+          choices: ["present", "absent"]
+          default: "present"
+          type: str
+        unified_job_template:
+          description:
+            - Name of unified job template to run in the workflow.
+            - Can be a job template, project sync, inventory source sync, etc.
+            - Omit if creating an approval node (not yet implemented).
+          type: dict
+          suboptions:
+            organization:
+              description:
+                - Name of key for use in model for organizational reference
+                - Only Valid and used if referencing a job template or project sync
+                - This parameter is mutually exclusive with suboption C(inventory).
+              type: dict
+              suboptions:
+                name:
+                  description:
+                    - The organization of the job template or project sync the node exists in.
+                    - Used for looking up the job template or project sync, not a direct model field.
+                  type: str
+            inventory:
+              description:
+                - Name of key for use in model for organizational reference
+                - Only Valid and used if referencing an inventory sync
+                - This parameter is mutually exclusive with suboption C(organization).
+              type: dict
+              suboptions:
+                organization:
+                  description:
+                    - Name of key for use in model for organizational reference
+                  type: dict
+                  suboptions:
+                    name:
+                      description:
+                        - The organization of the inventory the node exists in.
+                        - Used for looking up the job template or project, not a direct model field.
+                      type: str
+            name:
+              description:
+                - Name of unified job template to run in the workflow.
+                - Can be a job template, project, inventory source, etc.
+              type: str
+            description:
+              description:
+                - Optional description of this workflow approval template.
+              type: str
+            type:
+              description:
+                - Name of unified job template type to run in the workflow.
+                - Can be a job_template, project, inventory_source, workflow_approval.
+              type: str
+            timeout:
+              description:
+                - The amount of time (in seconds) to wait before Approval is canceled. A value of 0 means no timeout.
+                - Only Valid and used if referencing an Approval Node
+              default: 0
+              type: int
+        related:
+          description:
+            - Related items to this workflow node.
+            - Must include credentials, failure_nodes, always_nodes, success_nodes, even if empty.
+          type: dict
+          suboptions:
+            always_nodes:
+              description:
+                - Nodes that will run after this node completes.
+                - List of node identifiers.
+              type: list
+              suboptions:
+                identifier:
+                  description:
+                    - Identifier of Node that will run after this node completes given this option.
+                  elements: str
+            success_nodes:
+              description:
+                - Nodes that will run after this node on success.
+                - List of node identifiers.
+              type: list
+              suboptions:
+                identifier:
+                  description:
+                    - Identifier of Node that will run after this node completes given this option.
+                  elements: str
+            failure_nodes:
+              description:
+                - Nodes that will run after this node on failure.
+                - List of node identifiers.
+              type: list
+              suboptions:
+                identifier:
+                  description:
+                    - Identifier of Node that will run after this node completes given this option.
+                  elements: str
+            credentials:
+              description:
+                - Credentials to be applied to job as launch-time prompts.
+                - List of credential names.
+                - Uniqueness is not handled rigorously.
+              type: list
+              suboptions:
+                name:
+                  description:
+                    - Name Credentials to be applied to job as launch-time prompts.
+                  elements: str
+    destroy_current_schema:
+      description:
+        - Set in order to destroy current schema on the workflow.
+        - This option is used for full schema update, if not used, nodes not described in schema will persist and keep current associations and links.
+      type: bool
+      default: False
+
 extends_documentation_fragment: awx.awx.auth
 '''
 
@@ -154,17 +333,136 @@ EXAMPLES = '''
     description: created by Ansible Playbook
     organization: Default
 
+- name: Create a workflow job template with schema in template
+  awx.awx.tower_workflow_job_template:
+    name: example-workflow
+    inventory: Demo Inventory
+    extra_vars: {'foo': 'bar', 'another-foo': {'barz': 'bar2'}}
+    schema:
+      - identifier: node101
+        unified_job_template:
+          name: example-project
+          inventory:
+            organization:
+              name: Default
+          type: inventory_source
+        related:
+          success_nodes: []
+          failure_nodes:
+            - identifier: node201
+          always_nodes: []
+          credentials: []
+      - identifier: node201
+        unified_job_template:
+          organization:
+            name: Default
+          name: job template 1
+          type: job_template
+        credentials: []
+        related:
+          success_nodes:
+            - identifier: node301
+          failure_nodes: []
+          always_nodes: []
+          credentials: []
+      - identifier: node202
+        unified_job_template:
+          organization:
+            name: Default
+          name: example-project
+          type: project
+        related:
+          success_nodes: []
+          failure_nodes: []
+          always_nodes: []
+          credentials: []
+      - identifier: node301
+        all_parents_must_converge: false
+        unified_job_template:
+          organization:
+            name: Default
+          name: job template 2
+          type: job_template
+        related:
+          success_nodes: []
+          failure_nodes: []
+          always_nodes: []
+          credentials: []
+  register: result
+
 - name: Copy a workflow job template
   tower_workflow_job_template:
     name: copy-workflow
     copy_from: example-workflow
     organization: Foo
+
+- name: Create a workflow job template with schema in template
+  awx.awx.tower_workflow_job_template:
+    name: example-workflow
+    inventory: Demo Inventory
+    extra_vars: {'foo': 'bar', 'another-foo': {'barz': 'bar2'}}
+    schema:
+      - identifier: node101
+        unified_job_template:
+          name: example-project
+          inventory:
+            organization:
+              name: Default
+          type: inventory_source
+        related:
+          success_nodes: []
+          failure_nodes:
+            - identifier: node201
+          always_nodes: []
+          credentials: []
+      - identifier: node201
+        unified_job_template:
+          organization:
+            name: Default
+          name: job template 1
+          type: job_template
+        credentials: []
+        related:
+          success_nodes:
+            - identifier: node301
+          failure_nodes: []
+          always_nodes: []
+          credentials: []
+      - identifier: node202
+        unified_job_template:
+          organization:
+            name: Default
+          name: example-project
+          type: project
+        related:
+          success_nodes: []
+          failure_nodes: []
+          always_nodes: []
+          credentials: []
+      - identifier: node301
+        all_parents_must_converge: false
+        unified_job_template:
+          organization:
+            name: Default
+          name: job template 2
+          type: job_template
+        related:
+          success_nodes: []
+          failure_nodes: []
+          always_nodes: []
+          credentials: []
+  register: result
+
 '''
 
 from ..module_utils.tower_api import TowerAPIModule
 
 import json
 
+response = []
+
+response = []
+
 
 def update_survey(module, last_request):
     spec_endpoint = last_request.get('related', {}).get('survey_spec')
@@ -177,7 +475,185 @@ def update_survey(module, last_request):
         response = module.post_endpoint(spec_endpoint, **{'data': module.params.get('survey_spec')})
         if response['status_code'] != 200:
             module.fail_json(msg="Failed to update survey: {0}".format(response['json']['error']))
-    module.exit_json(**module.json_output)
+
+
+def create_schema_nodes(module, response, schema, workflow_id):
+    for workflow_node in schema:
+        workflow_node_fields = {}
+        search_fields = {}
+        association_fields = {}
+
+        # Lookup Job Template ID
+        if workflow_node['unified_job_template']['name']:
+            search_fields = {'name': workflow_node['unified_job_template']['name']}
+            if workflow_node['unified_job_template']['type'] is None:
+                module.fail_json(msg='Could not find unified job template type in schema {1}'.format(workflow_node))
+            if workflow_node['unified_job_template']['type'] == 'inventory_source':
+                # workflow_node['unified_job_template']['inventory']:
+                organization_id = module.resolve_name_to_id('organizations', workflow_node['unified_job_template']['inventory']['organization']['name'])
+                search_fields['organization'] = organization_id
+            elif workflow_node['unified_job_template']['type'] == 'workflow_approval':
+                pass
+            else:
+                # workflow_node['unified_job_template']['organization']:
+                organization_id = module.resolve_name_to_id('organizations', workflow_node['unified_job_template']['organization']['name'])
+                search_fields['organization'] = organization_id
+            unified_job_template = module.get_one('unified_job_templates', **{'data': search_fields})
+            if unified_job_template:
+                workflow_node_fields['unified_job_template'] = unified_job_template['id']
+            else:
+                if workflow_node['unified_job_template']['type'] != 'workflow_approval':
+                    module.fail_json(msg="Unable to Find unified_job_template: {0}".format(search_fields))
+
+        # Lookup Values for other fields
+
+        for field_name in (
+            'identifier',
+            'extra_data',
+            'scm_branch',
+            'job_type',
+            'job_tags',
+            'skip_tags',
+            'limit',
+            'diff_mode',
+            'verbosity',
+            'all_parents_must_converge',
+            'state',
+        ):
+            field_val = workflow_node.get(field_name)
+            if field_val:
+                workflow_node_fields[field_name] = field_val
+            if workflow_node['identifier']:
+                search_fields = {'identifier': workflow_node['identifier']}
+
+        # Set Search fields
+        search_fields['workflow_job_template'] = workflow_node_fields['workflow_job_template'] = workflow_id
+
+        # Attempt to look up an existing item based on the provided data
+        existing_item = module.get_one('workflow_job_template_nodes', **{'data': search_fields})
+
+        # Determine if state is present or absent.
+        state = True
+        if 'state' in workflow_node:
+            if workflow_node['state'] == 'absent':
+                state = False
+        if state:
+            response.append(
+                module.create_or_update_if_needed(
+                    existing_item,
+                    workflow_node_fields,
+                    endpoint='workflow_job_template_nodes',
+                    item_type='workflow_job_template_node',
+                    auto_exit=False,
+                )
+            )
+        else:
+            # If the state was absent we can let the module delete it if needed, the module will handle exiting from this
+            response.append(
+                module.delete_if_needed(
+                    existing_item,
+                    auto_exit=False,
+                )
+            )
+
+        # Start Approval Node creation process
+        if workflow_node['unified_job_template']['type'] == 'workflow_approval':
+            new_fields = {}
+
+            for field_name in (
+                'name',
+                'description',
+                'timeout',
+            ):
+                field_val = workflow_node['unified_job_template'].get(field_name)
+                if field_val:
+                    workflow_node_fields[field_name] = field_val
+
+            # Attempt to look up an existing item just created
+            workflow_job_template_node = module.get_one('workflow_job_template_nodes', **{'data': search_fields})
+            workflow_job_template_node_id = workflow_job_template_node['id']
+            existing_item = None
+            # Due to not able to lookup workflow_approval_templates, find the existing item in another place
+            if workflow_job_template_node['related'].get('unified_job_template') is not None:
+                existing_item = module.get_endpoint(workflow_job_template_node['related']['unified_job_template'])['json']
+            approval_endpoint = 'workflow_job_template_nodes/{0}/create_approval_template/'.format(workflow_job_template_node_id)
+
+            module.create_or_update_if_needed(
+                existing_item,
+                workflow_node_fields,
+                endpoint=approval_endpoint,
+                item_type='workflow_job_template_approval_node',
+                associations=association_fields,
+                auto_exit=False,
+            )
+
+
+def create_schema_nodes_association(module, response, schema, workflow_id):
+    for workflow_node in schema:
+        workflow_node_fields = {}
+        search_fields = {}
+        association_fields = {}
+
+        # Set Search fields
+        search_fields['workflow_job_template'] = workflow_node_fields['workflow_job_template'] = workflow_id
+
+        # Lookup Values for other fields
+        if workflow_node['identifier']:
+            workflow_node_fields['identifier'] = workflow_node['identifier']
+            search_fields['identifier'] = workflow_node['identifier']
+
+        # Attempt to look up an existing item based on the provided data
+        existing_item = module.get_one('workflow_job_template_nodes', **{'data': search_fields})
+
+        if 'state' in workflow_node:
+            if workflow_node['state'] == 'absent':
+                continue
+
+        if 'related' in workflow_node:
+            # Get id's for association fields
+            association_fields = {}
+
+            for association in ('always_nodes', 'success_nodes', 'failure_nodes', 'credentials'):
+                # Extract out information if it exists
+                # Test if it is defined, else move to next association.
+                if association in workflow_node['related']:
+                    id_list = []
+                    for sub_name in workflow_node['related'][association]:
+                        if association == 'credentials':
+                            endpoint = 'credentials'
+                            lookup_data = {'name': sub_name['name']}
+                        else:
+                            endpoint = 'workflow_job_template_nodes'
+                            lookup_data = {'identifier': sub_name['identifier']}
+                            lookup_data['workflow_job_template'] = workflow_id
+                        sub_obj = module.get_one(endpoint, **{'data': lookup_data})
+                        if sub_obj is None:
+                            module.fail_json(msg='Could not find {0} entry with name {1}'.format(association, sub_name))
+                        id_list.append(sub_obj['id'])
+                        temp = sub_obj['id']
+                    if id_list:
+                        association_fields[association] = id_list
+
+                    module.create_or_update_if_needed(
+                        existing_item,
+                        workflow_node_fields,
+                        endpoint='workflow_job_template_nodes',
+                        item_type='workflow_job_template_node',
+                        auto_exit=False,
+                        associations=association_fields,
+                    )
+
+
+def destroy_schema_nodes(module, response, workflow_id):
+    search_fields = {}
+
+    # Search for existing nodes.
+    search_fields['workflow_job_template'] = workflow_id
+    existing_items = module.get_all_endpoint('workflow_job_template_nodes', **{'data': search_fields})
+
+    # Loop through found fields
+    for workflow_node in existing_items['json']['results']:
+        response.append(module.delete_endpoint(workflow_node['url']))
 
 
 def main():
@@ -207,6 +683,8 @@ def main():
         notification_templates_success=dict(type="list", elements='str'),
         notification_templates_error=dict(type="list", elements='str'),
         notification_templates_approvals=dict(type="list", elements='str'),
+        schema=dict(type='list', elements='dict'),
+        destroy_current_schema=dict(type='bool', default=False),
         state=dict(choices=['present', 'absent'], default='present'),
     )
 
@@ -219,6 +697,12 @@ def main():
     copy_from = module.params.get('copy_from')
     state = module.params.get('state')
 
+    # Extract schema parameters
+    schema = None
+    if module.params.get('schema'):
+        schema = module.params.get('schema')
+    destroy_current_schema = module.params.get('destroy_current_schema')
+
     new_fields = {}
     search_fields = {}
 
@@ -341,8 +825,26 @@ def main():
         associations=association_fields,
         on_create=on_change,
         on_update=on_change,
+        auto_exit=False,
     )
 
+    # Get Workflow information in case one was just created.
+    existing_item = module.get_one('workflow_job_templates', name_or_id=name, **{'data': search_fields})
+    workflow_job_template_id = existing_item['id']
+    # Destroy current nodes if selected.
+    if destroy_current_schema:
+        destroy_schema_nodes(module, response, workflow_job_template_id)
+
+    # Work thorugh and lookup value for schema fields
+    if schema:
+        # Create Schema Nodes
+        create_schema_nodes(module, response, schema, workflow_job_template_id)
+        # Create Schema Associations
+        create_schema_nodes_association(module, response, schema, workflow_job_template_id)
+        module.json_output['schema_creation_data'] = response
+
+    module.exit_json(**module.json_output)
+
 
 if __name__ == '__main__':
     main()

awx_collection/test/awx/test_completeness.py

@@ -59,7 +59,7 @@ no_api_parameter_ok = {
     # Organization is how we are looking up job templates, Approval node is for workflow_approval_templates
     'tower_workflow_job_template_node': ['organization', 'approval_node'],
     # Survey is how we handle associations
-    'tower_workflow_job_template': ['survey_spec'],
+    'tower_workflow_job_template': ['survey_spec', 'destroy_current_schema'],
     # ad hoc commands support interval and timeout since its more like tower_job_launch
     'tower_ad_hoc_command': ['interval', 'timeout', 'wait'],
     # tower_group parameters to perserve hosts and children.

awx_collection/tests/integration/targets/tower_workflow_job_template/tasks/main.yml

@@ -14,6 +14,8 @@
     wfjt_name: "AWX-Collection-tests-tower_workflow_job_template-wfjt-{{ test_id }}"
     email_not: "AWX-Collection-tests-tower_job_template-email-not-{{ test_id }}"
     webhook_not: "AWX-Collection-tests-tower_notification_template-wehbook-not-{{ test_id }}"
+    project_inv: "AWX-Collection-tests-tower_inventory_source-inv-project-{{ lookup('password', '/dev/null chars=ascii_letters length=16') }}"
+    project_inv_source: "AWX-Collection-tests-tower_inventory_source-inv-source-project-{{ lookup('password', '/dev/null chars=ascii_letters length=16') }}"
 
 - name: Create an SCM Credential
   tower_credential:
@@ -72,6 +74,27 @@
     that:
       - "result is changed"
 
+- name: Add a Tower inventory
+  tower_inventory:
+    description: Test inventory
+    organization: Default
+    name: "{{ project_inv }}"
+
+- name: Create a source inventory
+  tower_inventory_source:
+    name: "{{ project_inv_source }}"
+    description: Source for Test inventory
+    inventory: "{{ project_inv }}"
+    source_project: "{{ demo_project_name }}"
+    source_path: "/inventories/inventory.ini"
+    overwrite: true
+    source: scm
+  register: result
+
+- assert:
+    that:
+      - "result is changed"
+
 - name: Create a Job Template
   tower_job_template:
     name: "{{ jt1_name }}"
@@ -324,6 +347,106 @@
       - "'Non_Existing_Organization' in result.msg"
       - "result.total_results == 0"
 
+- name: Create a workflow job template with schema in template
+  awx.awx.tower_workflow_job_template:
+    name: "{{ wfjt_name }}"
+    inventory: Demo Inventory
+    extra_vars: {'foo': 'bar', 'another-foo': {'barz': 'bar2'}}
+    schema:
+      - identifier: node101
+        unified_job_template:
+          name: "{{ project_inv_source }}"
+          inventory:
+            organization:
+              name: Default
+          type: inventory_source
+        related:
+          failure_nodes:
+            - identifier: node201
+      - identifier: node201
+        unified_job_template:
+          organization:
+            name: Default
+          name: "{{ jt1_name }}"
+          type: job_template
+        credentials: []
+        related:
+          success_nodes:
+            - identifier: node301
+      - identifier: node202
+        unified_job_template:
+          organization:
+            name: Default
+          name: "{{ demo_project_name }}"
+          type: project
+      - all_parents_must_converge: false
+        identifier: node301
+        unified_job_template:
+          organization:
+            name: Default
+          name: "{{ jt2_name }}"
+          type: job_template
+  register: result
+
+- assert:
+    that:
+      - "result is changed"
+
+- name: Kick off a workflow and wait for it
+  tower_workflow_launch:
+    workflow_template: "{{ wfjt_name }}"
+  ignore_errors: true
+  register: result
+
+- assert:
+    that:
+      - result is not failed
+      - "'id' in result['job_info']"
+
+- name: Destroy previous schema for one that fails
+  awx.awx.tower_workflow_job_template:
+    name: "{{ wfjt_name }}"
+    destroy_current_schema: true
+    schema:
+      - identifier: node101
+        unified_job_template:
+          organization:
+            name: Default
+          name: "{{ jt1_name }}"
+          type: job_template
+        credentials: []
+        related:
+          success_nodes:
+            - identifier: node201
+      - identifier: node201
+        unified_job_template:
+          name: "{{ project_inv_source }}"
+          inventory:
+            organization:
+              name: Default
+          type: inventory_source
+  register: result
+
+- name: Kick off a workflow and wait for it
+  tower_workflow_launch:
+    workflow_template: "{{ wfjt_name }}"
+  ignore_errors: true
+  register: result
+
+- assert:
+    that:
+      - result is failed
+
+- name: Delete a workflow job template
+  awx.awx.tower_workflow_job_template:
+    name: "{{ wfjt_name }}"
+    state: absent
+  register: result
+
+- assert:
+    that:
+      - "result is changed"
+
 - name: Delete the Job Template
   tower_job_template:
     name: "{{ jt1_name }}"
@@ -352,6 +475,28 @@
     that:
       - "result is changed"
 
+- name: Delete the inventory source
+  tower_inventory_source:
+    name: "{{ project_inv_source }}"
+    inventory: "{{ project_inv }}"
+    source: scm
+    state: absent
+
+- assert:
+    that:
+      - "result is changed"
+
+- name: Delete the inventory
+  tower_inventory:
+    description: Test inventory
+    organization: Default
+    name: "{{ project_inv }}"
+    state: absent
+
+- assert:
+    that:
+      - "result is changed"
+
 - name: Delete the Demo Project
   tower_project:
     name: "{{ demo_project_name }}"