From cf24c81b3e8767b4ea98c700eb92014de4690455 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Tue, 10 Dec 2019 15:17:34 -0500 Subject: [PATCH 01/24] updated syntax from python2 to 3 --- awx/main/models/unified_jobs.py | 7 +++++++ awx/main/models/workflow.py | 2 +- awx/main/scheduler/dag_simple.py | 4 ++-- awx/main/scheduler/dag_workflow.py | 2 ++ .../awxkit/api/pages/workflow_job_template_nodes.py | 12 +++++++++--- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 1617014540..2719d59c32 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -698,6 +698,12 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique 'Credential', related_name='%(class)ss', ) + # convergence_behavior = models.TextField( + # blank=True, + # default='AND', + # editable=True, + # help_text=_('The behavior by a convergence node') + # ) def get_absolute_url(self, request=None): RealClass = self.get_real_instance_class() @@ -1445,3 +1451,4 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique @property def is_containerized(self): return False + diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 100ba1c323..3326356f72 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -129,7 +129,7 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig): class WorkflowJobTemplateNode(WorkflowNodeBase): FIELDS_TO_PRESERVE_AT_COPY = [ 'unified_job_template', 'workflow_job_template', 'success_nodes', 'failure_nodes', - 'always_nodes', 'credentials', 'inventory', 'extra_data', 'survey_passwords', + 'always_nodes', 'parent_nodes', 'credentials', 'inventory', 'extra_data', 'survey_passwords', 'char_prompts' ] REENCRYPTION_BLACKLIST_AT_COPY = ['extra_data', 'survey_passwords'] diff --git a/awx/main/scheduler/dag_simple.py b/awx/main/scheduler/dag_simple.py index 71f1ff73c2..b2741e3538 100644 --- a/awx/main/scheduler/dag_simple.py +++ b/awx/main/scheduler/dag_simple.py @@ -89,8 +89,8 @@ class SimpleDAG(object): run_status(n['node_object']), color ) - for label, edges in self.node_from_edges_by_label.iteritems(): - for from_node, to_nodes in edges.iteritems(): + for label, edges in self.node_from_edges_by_label.items(): + for from_node, to_nodes in edges.items(): for to_node in to_nodes: doc += "%s -> %s [ label=\"%s\" ];\n" % ( run_status(self.nodes[from_node]['node_object']), diff --git a/awx/main/scheduler/dag_workflow.py b/awx/main/scheduler/dag_workflow.py index 8676630247..e4b926c853 100644 --- a/awx/main/scheduler/dag_workflow.py +++ b/awx/main/scheduler/dag_workflow.py @@ -96,6 +96,7 @@ class WorkflowDAG(SimpleDAG): else: if self._are_relevant_parents_finished(n): nodes_found.append(n) + #BECCAH TODO somewhere around here add in ANY and ALL logic return [n['node_object'] for n in nodes_found] def cancel_node_jobs(self): @@ -185,6 +186,7 @@ class WorkflowDAG(SimpleDAG): Return a boolean ''' def _should_mark_node_dnr(self, node, parent_nodes): + #BECCAH TODO Gonna have to update this too for p in parent_nodes: if p.do_not_run is True: pass diff --git a/awxkit/awxkit/api/pages/workflow_job_template_nodes.py b/awxkit/awxkit/api/pages/workflow_job_template_nodes.py index 4a36968f54..f33987ca3a 100644 --- a/awxkit/awxkit/api/pages/workflow_job_template_nodes.py +++ b/awxkit/awxkit/api/pages/workflow_job_template_nodes.py @@ -83,11 +83,17 @@ class WorkflowJobTemplateNode(HasCreate, base.Base): def add_always_node(self, unified_job_template): return self._add_node(self.related.always_nodes, unified_job_template) - def add_failure_node(self, unified_job_template): + def add_any_successes_node(self, unified_job_template): + return self._add_node(self.related.success_nodes, unified_job_template) + + def add_all_successes_node(self, unified_job_template): + return self._add_node(self.related.success_nodes, unified_job_template) + + def add_any_failure_node(self, unified_job_template): return self._add_node(self.related.failure_nodes, unified_job_template) - def add_success_node(self, unified_job_template): - return self._add_node(self.related.success_nodes, unified_job_template) + def add_all_failures_node(self, unified_job_template): + return self._add_node(self.related.failure_nodes, unified_job_template) def add_credential(self, credential): with suppress(exc.NoContent): From 4c35adad6c4396aa25136f9016b6b7687c69b931 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Mon, 6 Jan 2020 16:36:39 -0500 Subject: [PATCH 02/24] added logic to include workflow convergence nodes to nodes to run or not run based on their parents successful statuses --- awx/main/scheduler/dag_workflow.py | 64 ++++++++++++++++++------------ 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/awx/main/scheduler/dag_workflow.py b/awx/main/scheduler/dag_workflow.py index e4b926c853..de07751d3b 100644 --- a/awx/main/scheduler/dag_workflow.py +++ b/awx/main/scheduler/dag_workflow.py @@ -55,7 +55,7 @@ class WorkflowDAG(SimpleDAG): def _are_relevant_parents_finished(self, node): obj = node['node_object'] - parent_nodes = [p['node_object'] for p in self.get_dependents(obj)] + parent_nodes = [p['node_object'] for p in self.get_parents(obj)] for p in parent_nodes: if p.do_not_run is True: continue @@ -73,7 +73,6 @@ class WorkflowDAG(SimpleDAG): nodes = self.get_root_nodes() nodes_found = [] node_ids_visited = set() - for index, n in enumerate(nodes): obj = n['node_object'] if obj.id in node_ids_visited: @@ -82,21 +81,34 @@ class WorkflowDAG(SimpleDAG): if obj.do_not_run is True: continue - - if obj.job: - if obj.job.status in ['failed', 'error', 'canceled']: - nodes.extend(self.get_dependencies(obj, 'failure_nodes') + - self.get_dependencies(obj, 'always_nodes')) - elif obj.job.status == 'successful': - nodes.extend(self.get_dependencies(obj, 'success_nodes') + - self.get_dependencies(obj, 'always_nodes')) - elif obj.unified_job_template is None: - nodes.extend(self.get_dependencies(obj, 'failure_nodes') + - self.get_dependencies(obj, 'always_nodes')) else: - if self._are_relevant_parents_finished(n): - nodes_found.append(n) - #BECCAH TODO somewhere around here add in ANY and ALL logic + if obj.all_parents_must_converge is True: + if self._are_relevant_parents_finished(n): + # if the current node is a convergence node and all the + # relevant parents are finished then the node should run + parent_nodes = [p['node_object'] for p in self.get_parents(obj)] + successful_convergence = True + for p in parent_nodes: + if obj not in self.get_children(p, p.job.status): + # if the child list doesn't include the obj, then the parent didn't + # meet the criteria needed to run the child, meaning it's a DNR + successful_convergence = False + if successful_convergence == True: + nodes_found.append(n) + elif obj.all_parents_must_converge is False: + if obj.job: + if obj.job.status in ['failed', 'error', 'canceled']: + nodes.extend(self.get_children(obj, 'failure_nodes') + + self.get_children(obj, 'always_nodes')) + elif obj.job.status == 'successful': + nodes.extend(self.get_children(obj, 'success_nodes') + + self.get_children(obj, 'always_nodes')) + elif obj.unified_job_template is None: + nodes.extend(self.get_children(obj, 'failure_nodes') + + self.get_children(obj, 'always_nodes')) + else: + if self._are_relevant_parents_finished(n) is True: + nodes_found.append(n) return [n['node_object'] for n in nodes_found] def cancel_node_jobs(self): @@ -136,8 +148,8 @@ class WorkflowDAG(SimpleDAG): for node in failed_nodes: obj = node['node_object'] - if (len(self.get_dependencies(obj, 'failure_nodes')) + - len(self.get_dependencies(obj, 'always_nodes'))) == 0: + if (len(self.get_children(obj, 'failure_nodes')) + + len(self.get_children(obj, 'always_nodes'))) == 0: if obj.unified_job_template is None: res = True failed_unified_job_template_node_ids.append(str(obj.id)) @@ -192,18 +204,18 @@ class WorkflowDAG(SimpleDAG): pass elif p.job: if p.job.status == 'successful': - if node in (self.get_dependencies(p, 'success_nodes') + - self.get_dependencies(p, 'always_nodes')): + if node in (self.get_children(p, 'success_nodes') + + self.get_children(p, 'always_nodes')): return False elif p.job.status in ['failed', 'error', 'canceled']: - if node in (self.get_dependencies(p, 'failure_nodes') + - self.get_dependencies(p, 'always_nodes')): + if node in (self.get_children(p, 'failure_nodes') + + self.get_children(p, 'always_nodes')): return False else: return False elif p.do_not_run is False and p.unified_job_template is None: - if node in (self.get_dependencies(p, 'failure_nodes') + - self.get_dependencies(p, 'always_nodes')): + if node in (self.get_children(p, 'failure_nodes') + + self.get_children(p, 'always_nodes')): return False else: return False @@ -217,10 +229,10 @@ class WorkflowDAG(SimpleDAG): obj = node['node_object'] if obj.do_not_run is False and not obj.job and node not in root_nodes: - parent_nodes = [p['node_object'] for p in self.get_dependents(obj)] + parent_nodes = [p['node_object'] for p in self.get_parents(obj)] if self._are_all_nodes_dnr_decided(parent_nodes): if self._should_mark_node_dnr(node, parent_nodes): obj.do_not_run = True nodes_marked_do_not_run.append(node) - return [n['node_object'] for n in nodes_marked_do_not_run] + return [n['node_object'] for n in nodes_marked_do_not_run] \ No newline at end of file From 780f104ab2f98b6410d10682ae1080506780ae8d Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Mon, 6 Jan 2020 16:39:38 -0500 Subject: [PATCH 03/24] shifted from dependants/dependencies to children/parents for clarity in function names, also added in toggle logic --- awx/main/models/workflow.py | 7 ++++++- awx/main/scheduler/dag_simple.py | 20 +++++++++---------- .../api/pages/workflow_job_template_nodes.py | 12 +++-------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 3326356f72..de2767e2d7 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -129,7 +129,7 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig): class WorkflowJobTemplateNode(WorkflowNodeBase): FIELDS_TO_PRESERVE_AT_COPY = [ 'unified_job_template', 'workflow_job_template', 'success_nodes', 'failure_nodes', - 'always_nodes', 'parent_nodes', 'credentials', 'inventory', 'extra_data', 'survey_passwords', + 'always_nodes', 'credentials', 'inventory', 'extra_data', 'survey_passwords', 'char_prompts' ] REENCRYPTION_BLACKLIST_AT_COPY = ['extra_data', 'survey_passwords'] @@ -208,6 +208,11 @@ class WorkflowJobNode(WorkflowNodeBase): "semantics will mark this True if the node is in a path that will " "decidedly not be ran. A value of False means the node may not run."), ) + all_parents_must_converge = models.BooleanField( + default=False, + help_text=_("If enabled then the node will only run if all of the parent nodes " + "have met the criteria to reach this node") + ) def get_absolute_url(self, request=None): return reverse('api:workflow_job_node_detail', kwargs={'pk': self.pk}, request=request) diff --git a/awx/main/scheduler/dag_simple.py b/awx/main/scheduler/dag_simple.py index b2741e3538..af6a819914 100644 --- a/awx/main/scheduler/dag_simple.py +++ b/awx/main/scheduler/dag_simple.py @@ -140,36 +140,36 @@ class SimpleDAG(object): def find_ord(self, obj): return self.node_obj_to_node_index.get(obj, None) - def _get_dependencies_by_label(self, node_index, label): + def _get_children_by_label(self, node_index, label): return [self.nodes[index] for index in self.node_from_edges_by_label.get(label, {}) .get(node_index, [])] - def get_dependencies(self, obj, label=None): + def get_children(self, obj, label=None): this_ord = self.find_ord(obj) nodes = [] if label: - return self._get_dependencies_by_label(this_ord, label) + return self._get_children_by_label(this_ord, label) else: nodes = [] for l in self.node_from_edges_by_label.keys(): - nodes.extend(self._get_dependencies_by_label(this_ord, l)) + nodes.extend(self._get_children_by_label(this_ord, l)) return nodes - def _get_dependents_by_label(self, node_index, label): + def _get_parents_by_label(self, node_index, label): return [self.nodes[index] for index in self.node_to_edges_by_label.get(label, {}) .get(node_index, [])] - def get_dependents(self, obj, label=None): + def get_parents(self, obj, label=None): this_ord = self.find_ord(obj) nodes = [] if label: - return self._get_dependents_by_label(this_ord, label) + return self._get_parents_by_label(this_ord, label) else: nodes = [] for l in self.node_to_edges_by_label.keys(): - nodes.extend(self._get_dependents_by_label(this_ord, l)) + nodes.extend(self._get_parents_by_label(this_ord, l)) return nodes def get_root_nodes(self): @@ -188,7 +188,7 @@ class SimpleDAG(object): while stack: node_obj = stack.pop() - children = [node['node_object'] for node in self.get_dependencies(node_obj)] + children = [node['node_object'] for node in self.get_children(node_obj)] children_to_add = list(filter(lambda node_obj: node_obj not in node_objs_visited, children)) if children_to_add: @@ -212,7 +212,7 @@ class SimpleDAG(object): if obj.id in obj_ids_processed: return - for child in self.get_dependencies(obj): + for child in self.get_children(obj): visit(child) obj_ids_processed.add(obj.id) nodes_sorted.appendleft(node) diff --git a/awxkit/awxkit/api/pages/workflow_job_template_nodes.py b/awxkit/awxkit/api/pages/workflow_job_template_nodes.py index f33987ca3a..4a36968f54 100644 --- a/awxkit/awxkit/api/pages/workflow_job_template_nodes.py +++ b/awxkit/awxkit/api/pages/workflow_job_template_nodes.py @@ -83,17 +83,11 @@ class WorkflowJobTemplateNode(HasCreate, base.Base): def add_always_node(self, unified_job_template): return self._add_node(self.related.always_nodes, unified_job_template) - def add_any_successes_node(self, unified_job_template): - return self._add_node(self.related.success_nodes, unified_job_template) - - def add_all_successes_node(self, unified_job_template): - return self._add_node(self.related.success_nodes, unified_job_template) - - def add_any_failure_node(self, unified_job_template): + def add_failure_node(self, unified_job_template): return self._add_node(self.related.failure_nodes, unified_job_template) - def add_all_failures_node(self, unified_job_template): - return self._add_node(self.related.failure_nodes, unified_job_template) + def add_success_node(self, unified_job_template): + return self._add_node(self.related.success_nodes, unified_job_template) def add_credential(self, credential): with suppress(exc.NoContent): From f7f648b95641056fd42689eb4044e6ae2b191765 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 8 Jan 2020 16:17:32 -0500 Subject: [PATCH 04/24] included all_parents_must_converge in the get_workflow_job_fieldnames so that the true/false is copied into the job node and not just in the template node. Also added in the migration for the DB, also relocated logic from bfs_nodes_to_run down into mark_dnr_nodes to prevent nodes not being marked as DNR but not being marked to run, causing them to run anyways --- awx/api/serializers.py | 6 +-- ...02_v370_workflow_convergence_api_toggle.py | 23 ++++++++ awx/main/models/unified_jobs.py | 7 --- awx/main/models/workflow.py | 13 ++--- awx/main/scheduler/dag_workflow.py | 54 +++++++++---------- 5 files changed, 57 insertions(+), 46 deletions(-) create mode 100644 awx/main/migrations/0102_v370_workflow_convergence_api_toggle.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index cda572a4ee..81ff15f0f7 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -3677,7 +3677,7 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer): class Meta: model = WorkflowJobTemplateNode fields = ('*', 'workflow_job_template', '-name', '-description', 'id', 'url', 'related', - 'unified_job_template', 'success_nodes', 'failure_nodes', 'always_nodes',) + 'unified_job_template', 'success_nodes', 'failure_nodes', 'always_nodes', 'all_parents_must_converge',) def get_related(self, obj): res = super(WorkflowJobTemplateNodeSerializer, self).get_related(obj) @@ -3716,8 +3716,8 @@ class WorkflowJobNodeSerializer(LaunchConfigurationBaseSerializer): class Meta: model = WorkflowJobNode fields = ('*', 'job', 'workflow_job', '-name', '-description', 'id', 'url', 'related', - 'unified_job_template', 'success_nodes', 'failure_nodes', 'always_nodes', - 'do_not_run',) + 'unified_job_template', 'success_nodes', 'failure_nodes', 'always_nodes', + 'all_parents_must_converge', 'do_not_run',) def get_related(self, obj): res = super(WorkflowJobNodeSerializer, self).get_related(obj) diff --git a/awx/main/migrations/0102_v370_workflow_convergence_api_toggle.py b/awx/main/migrations/0102_v370_workflow_convergence_api_toggle.py new file mode 100644 index 0000000000..340949e8f6 --- /dev/null +++ b/awx/main/migrations/0102_v370_workflow_convergence_api_toggle.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.4 on 2020-01-07 19:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0100_v370_projectupdate_job_tags'), + ] + + operations = [ + migrations.AddField( + model_name='workflowjobnode', + name='all_parents_must_converge', + field=models.BooleanField(default=False, help_text='If enabled then the node will only run if all of the parent nodes have met the criteria to reach this node', verbose_name='self'), + ), + migrations.AddField( + model_name='workflowjobtemplatenode', + name='all_parents_must_converge', + field=models.BooleanField(default=False, help_text='If enabled then the node will only run if all of the parent nodes have met the criteria to reach this node', verbose_name='self'), + ), + ] diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 2719d59c32..1617014540 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -698,12 +698,6 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique 'Credential', related_name='%(class)ss', ) - # convergence_behavior = models.TextField( - # blank=True, - # default='AND', - # editable=True, - # help_text=_('The behavior by a convergence node') - # ) def get_absolute_url(self, request=None): RealClass = self.get_real_instance_class() @@ -1451,4 +1445,3 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique @property def is_containerized(self): return False - diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index de2767e2d7..51733e10d4 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -79,6 +79,12 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig): symmetrical=False, related_name='%(class)ss_always', ) + all_parents_must_converge = models.BooleanField( + 'self', + default=False, + help_text=_("If enabled then the node will only run if all of the parent nodes " + "have met the criteria to reach this node") + ) unified_job_template = models.ForeignKey( 'UnifiedJobTemplate', related_name='%(class)ss', @@ -102,7 +108,7 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig): ''' return ['workflow_job', 'unified_job_template', 'extra_data', 'survey_passwords', - 'inventory', 'credentials', 'char_prompts'] + 'inventory', 'credentials', 'char_prompts', 'all_parents_must_converge'] def create_workflow_job_node(self, **kwargs): ''' @@ -208,11 +214,6 @@ class WorkflowJobNode(WorkflowNodeBase): "semantics will mark this True if the node is in a path that will " "decidedly not be ran. A value of False means the node may not run."), ) - all_parents_must_converge = models.BooleanField( - default=False, - help_text=_("If enabled then the node will only run if all of the parent nodes " - "have met the criteria to reach this node") - ) def get_absolute_url(self, request=None): return reverse('api:workflow_job_node_detail', kwargs={'pk': self.pk}, request=request) diff --git a/awx/main/scheduler/dag_workflow.py b/awx/main/scheduler/dag_workflow.py index de07751d3b..25e94b0a73 100644 --- a/awx/main/scheduler/dag_workflow.py +++ b/awx/main/scheduler/dag_workflow.py @@ -78,37 +78,23 @@ class WorkflowDAG(SimpleDAG): if obj.id in node_ids_visited: continue node_ids_visited.add(obj.id) - + # import sdb + # sdb.set_trace() if obj.do_not_run is True: continue + elif obj.job: + if obj.job.status in ['failed', 'error', 'canceled']: + nodes.extend(self.get_children(obj, 'failure_nodes') + + self.get_children(obj, 'always_nodes')) + elif obj.job.status == 'successful': + nodes.extend(self.get_children(obj, 'success_nodes') + + self.get_children(obj, 'always_nodes')) + elif obj.unified_job_template is None: + nodes.extend(self.get_children(obj, 'failure_nodes') + + self.get_children(obj, 'always_nodes')) else: - if obj.all_parents_must_converge is True: - if self._are_relevant_parents_finished(n): - # if the current node is a convergence node and all the - # relevant parents are finished then the node should run - parent_nodes = [p['node_object'] for p in self.get_parents(obj)] - successful_convergence = True - for p in parent_nodes: - if obj not in self.get_children(p, p.job.status): - # if the child list doesn't include the obj, then the parent didn't - # meet the criteria needed to run the child, meaning it's a DNR - successful_convergence = False - if successful_convergence == True: - nodes_found.append(n) - elif obj.all_parents_must_converge is False: - if obj.job: - if obj.job.status in ['failed', 'error', 'canceled']: - nodes.extend(self.get_children(obj, 'failure_nodes') + - self.get_children(obj, 'always_nodes')) - elif obj.job.status == 'successful': - nodes.extend(self.get_children(obj, 'success_nodes') + - self.get_children(obj, 'always_nodes')) - elif obj.unified_job_template is None: - nodes.extend(self.get_children(obj, 'failure_nodes') + - self.get_children(obj, 'always_nodes')) - else: - if self._are_relevant_parents_finished(n) is True: - nodes_found.append(n) + if self._are_relevant_parents_finished(n) is True: + nodes_found.append(n) return [n['node_object'] for n in nodes_found] def cancel_node_jobs(self): @@ -198,7 +184,6 @@ class WorkflowDAG(SimpleDAG): Return a boolean ''' def _should_mark_node_dnr(self, node, parent_nodes): - #BECCAH TODO Gonna have to update this too for p in parent_nodes: if p.do_not_run is True: pass @@ -234,5 +219,14 @@ class WorkflowDAG(SimpleDAG): if self._should_mark_node_dnr(node, parent_nodes): obj.do_not_run = True nodes_marked_do_not_run.append(node) + if obj.all_parents_must_converge: + if self._are_relevant_parents_finished(node): + # if the current node is a convergence node and all the + # parents are finished then check to see if all parents + # met their success criteria to run the convergence child + parent_nodes = [p['node_object'] for p in self.get_parents(obj)] + if not all(obj in self.get_children(p, p.job.status) for p in parent_nodes): + obj.do_not_run = True + nodes_marked_do_not_run.append(node) - return [n['node_object'] for n in nodes_marked_do_not_run] \ No newline at end of file + return [n['node_object'] for n in nodes_marked_do_not_run] From b2c33e320470e41a8d0db947b7a171b73a68f88f Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 8 Jan 2020 17:29:20 -0500 Subject: [PATCH 05/24] redid migration dependency --- .../migrations/0102_v370_workflow_convergence_api_toggle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/main/migrations/0102_v370_workflow_convergence_api_toggle.py b/awx/main/migrations/0102_v370_workflow_convergence_api_toggle.py index 340949e8f6..71dc4083db 100644 --- a/awx/main/migrations/0102_v370_workflow_convergence_api_toggle.py +++ b/awx/main/migrations/0102_v370_workflow_convergence_api_toggle.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.4 on 2020-01-07 19:24 +# Generated by Django 2.2.4 on 2020-01-08 22:11 from django.db import migrations, models @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('main', '0100_v370_projectupdate_job_tags'), + ('main', '0101_v370_generate_new_uuids_for_iso_nodes'), ] operations = [ From 2d3172f648dae6b834db5c0a84b6df40b59891fc Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Thu, 9 Jan 2020 19:05:37 -0500 Subject: [PATCH 06/24] added in support for existing workflow unit tests --- awx/main/tests/unit/models/test_workflow_unit.py | 2 ++ awx/main/tests/unit/scheduler/test_dag_workflow.py | 1 + 2 files changed, 3 insertions(+) diff --git a/awx/main/tests/unit/models/test_workflow_unit.py b/awx/main/tests/unit/models/test_workflow_unit.py index 5f53d97a0b..1dc4003ef0 100644 --- a/awx/main/tests/unit/models/test_workflow_unit.py +++ b/awx/main/tests/unit/models/test_workflow_unit.py @@ -171,6 +171,7 @@ class TestWorkflowJobCreate: with mocker.patch('awx.main.models.WorkflowJobNode.objects.create', mock_create): wfjt_node_no_prompts.create_workflow_job_node(workflow_job=workflow_job_unit) mock_create.assert_called_once_with( + all_parents_must_converge=False, extra_data={}, survey_passwords={}, char_prompts=wfjt_node_no_prompts.char_prompts, @@ -185,6 +186,7 @@ class TestWorkflowJobCreate: workflow_job=workflow_job_unit ) mock_create.assert_called_once_with( + all_parents_must_converge=False, extra_data={}, survey_passwords={}, char_prompts=wfjt_node_with_prompts.char_prompts, diff --git a/awx/main/tests/unit/scheduler/test_dag_workflow.py b/awx/main/tests/unit/scheduler/test_dag_workflow.py index 7f85ba2f85..6a1d81386d 100644 --- a/awx/main/tests/unit/scheduler/test_dag_workflow.py +++ b/awx/main/tests/unit/scheduler/test_dag_workflow.py @@ -19,6 +19,7 @@ class WorkflowNode(object): self.job = job self.do_not_run = do_not_run self.unified_job_template = unified_job_template + self.all_parents_must_converge = False @pytest.fixture From 70cf4cf5d424ca994ddd1ab8cc5bb1aaa117b922 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Mon, 13 Jan 2020 13:53:07 -0500 Subject: [PATCH 07/24] added in handling for a parent being DNR so status is only checked if the parent isn't a DNR parent (in which case the parent has no status, which was breaking the logic) also edited a comment and added in a DNR check that @alancoding suggested to cut out duplicates in the DAG list --- awx/main/scheduler/dag_workflow.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/awx/main/scheduler/dag_workflow.py b/awx/main/scheduler/dag_workflow.py index 25e94b0a73..4777d68948 100644 --- a/awx/main/scheduler/dag_workflow.py +++ b/awx/main/scheduler/dag_workflow.py @@ -219,13 +219,14 @@ class WorkflowDAG(SimpleDAG): if self._should_mark_node_dnr(node, parent_nodes): obj.do_not_run = True nodes_marked_do_not_run.append(node) - if obj.all_parents_must_converge: + if not obj.do_not_run and obj.all_parents_must_converge: if self._are_relevant_parents_finished(node): # if the current node is a convergence node and all the # parents are finished then check to see if all parents - # met their success criteria to run the convergence child + # met the needed criteria to run the convergence child + # (i.e. parent must fail, parent must succeed) parent_nodes = [p['node_object'] for p in self.get_parents(obj)] - if not all(obj in self.get_children(p, p.job.status) for p in parent_nodes): + if any(p.do_not_run for p in parent_nodes) or not all(obj in self.get_children(p, p.job.status)for p in parent_nodes): obj.do_not_run = True nodes_marked_do_not_run.append(node) From 987fc26537899885478d913078572fb660086676 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Mon, 13 Jan 2020 14:52:58 -0500 Subject: [PATCH 08/24] Add any/all option for workflow node convergence --- .../features/templates/templates.strings.js | 3 ++- .../forms/workflow-node-form.controller.js | 7 +++++- .../forms/workflow-node-form.partial.html | 22 +++++++++++++++++++ .../workflow-maker.controller.js | 4 +++- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/awx/ui/client/features/templates/templates.strings.js b/awx/ui/client/features/templates/templates.strings.js index 4c53a69acd..750a973ba1 100644 --- a/awx/ui/client/features/templates/templates.strings.js +++ b/awx/ui/client/features/templates/templates.strings.js @@ -153,7 +153,8 @@ function TemplatesStrings (BaseString) { TIMED_OUT: t.s('APPROVAL TIMED OUT'), TIMEOUT: t.s('Timeout'), APPROVED: t.s('APPROVED'), - DENIED: t.s('DENIED') + DENIED: t.s('DENIED'), + ALL_PARENTS_MUST_CONVERGE: t.s('All parents must converge before running'), }; } diff --git a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.controller.js b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.controller.js index 136b2e6cdf..64bce33067 100644 --- a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.controller.js +++ b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.controller.js @@ -500,6 +500,10 @@ export default ['$scope', 'TemplatesService', 'JobTemplateModel', 'PromptService type: 'workflow_job_template,job_template' }; + $scope.all_parents_must_converge = _.get( + $scope, ['nodeConfig', 'node', 'all_parents_must_converge'], + _.get($scope, ['nodeConfig', 'node', 'originalNodeObject', 'all_parents_must_converge'], false) + ); $scope.wf_maker_templates = []; $scope.wf_maker_template_dataset = {}; @@ -617,7 +621,8 @@ export default ['$scope', 'TemplatesService', 'JobTemplateModel', 'PromptService $scope.confirmNodeForm = () => { const nodeFormData = { - edgeType: $scope.edgeType + edgeType: $scope.edgeType, + all_parents_must_converge: $scope.all_parents_must_converge, }; if ($scope.activeTab === "approval") { diff --git a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html index b79ca9e79b..223897948f 100644 --- a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html +++ b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html @@ -183,6 +183,28 @@ +
+ +
+
+
+ +
+
+
+
{ if (node.id === nodeId) { From 86a39938fe420b367c6135d720168fa0d86d9057 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Fri, 17 Jan 2020 01:35:26 -0500 Subject: [PATCH 09/24] fixed issue where successful convergence wasn't being met due to the not quite correct leveraging of get_children --- awx/main/scheduler/dag_workflow.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/awx/main/scheduler/dag_workflow.py b/awx/main/scheduler/dag_workflow.py index 4777d68948..b50c6fa96b 100644 --- a/awx/main/scheduler/dag_workflow.py +++ b/awx/main/scheduler/dag_workflow.py @@ -78,8 +78,6 @@ class WorkflowDAG(SimpleDAG): if obj.id in node_ids_visited: continue node_ids_visited.add(obj.id) - # import sdb - # sdb.set_trace() if obj.do_not_run is True: continue elif obj.job: @@ -213,21 +211,32 @@ class WorkflowDAG(SimpleDAG): for node in self.sort_nodes_topological(): obj = node['node_object'] - if obj.do_not_run is False and not obj.job and node not in root_nodes: + if obj.do_not_run is False and not obj.job and node not in root_nodes and not obj.all_parents_must_converge: parent_nodes = [p['node_object'] for p in self.get_parents(obj)] if self._are_all_nodes_dnr_decided(parent_nodes): if self._should_mark_node_dnr(node, parent_nodes): obj.do_not_run = True nodes_marked_do_not_run.append(node) - if not obj.do_not_run and obj.all_parents_must_converge: + if obj.do_not_run is False and obj.all_parents_must_converge: if self._are_relevant_parents_finished(node): # if the current node is a convergence node and all the # parents are finished then check to see if all parents # met the needed criteria to run the convergence child # (i.e. parent must fail, parent must succeed) parent_nodes = [p['node_object'] for p in self.get_parents(obj)] - if any(p.do_not_run for p in parent_nodes) or not all(obj in self.get_children(p, p.job.status)for p in parent_nodes): + if any(p.do_not_run for p in parent_nodes): obj.do_not_run = True nodes_marked_do_not_run.append(node) + else: + for p in parent_nodes: + if p.job.status == "successful": + child_nodes = (c['node_object'] for c in self.get_children(p, "success_nodes")) + elif p.job.status == "failure": + child_nodes = (c['node_object'] for c in self.get_children(p, "failure_nodes")) + else: + child_nodes = (c['node_object'] for c in self.get_children(p, "always_nodes")) + if not any(obj == x for x in child_nodes): + obj.do_not_run = True + nodes_marked_do_not_run.append(node) return [n['node_object'] for n in nodes_marked_do_not_run] From bbb4701fa92b7c8776b1e1ca3cd7576feb48251e Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Tue, 21 Jan 2020 12:30:33 -0500 Subject: [PATCH 10/24] Don't show any/all convergence option on root node --- .../forms/workflow-node-form.partial.html | 2 +- .../workflow-maker/workflow-maker.controller.js | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html index 223897948f..cdc6c966aa 100644 --- a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html +++ b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html @@ -183,7 +183,7 @@
-
+
diff --git a/awx/ui/client/src/templates/workflows/workflow-maker/workflow-maker.controller.js b/awx/ui/client/src/templates/workflows/workflow-maker/workflow-maker.controller.js index 95dbe04d2f..17d3d0b05e 100644 --- a/awx/ui/client/src/templates/workflows/workflow-maker/workflow-maker.controller.js +++ b/awx/ui/client/src/templates/workflows/workflow-maker/workflow-maker.controller.js @@ -720,6 +720,15 @@ export default ['$scope', 'TemplatesService', /* EDIT NODE FUNCTIONS */ $scope.startEditNode = (nodeToEdit) => { + // Determine if this is a root node + let editNodeIsRoot = false; + $scope.graphState.arrayOfLinksForChart.forEach((link) => { + if (link.source.id === 1 && link.target.id === nodeIdToChartNodeIdMapping[nodeToEdit.id]) { + editNodeIsRoot = true; + return; + } + }); + $scope.workflowChangesStarted = true; if ($scope.linkConfig) { $scope.cancelLinkForm(); @@ -733,7 +742,8 @@ export default ['$scope', 'TemplatesService', $scope.nodeConfig = { mode: "edit", nodeId: nodeToEdit.id, - node: nodeRef[nodeToEdit.id] + node: nodeRef[nodeToEdit.id], + editNodeIsRoot, }; $scope.graphState.nodeBeingEdited = nodeToEdit.id; From 4fe9e5da140de7a742508f01ccd61627268c7715 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Tue, 21 Jan 2020 13:31:05 -0500 Subject: [PATCH 11/24] Use select for any/all convergence choice --- .../features/templates/templates.strings.js | 4 ++- .../forms/workflow-node-form.controller.js | 20 ++++++++++-- .../forms/workflow-node-form.partial.html | 32 ++++++++----------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/awx/ui/client/features/templates/templates.strings.js b/awx/ui/client/features/templates/templates.strings.js index 750a973ba1..68c97c57f7 100644 --- a/awx/ui/client/features/templates/templates.strings.js +++ b/awx/ui/client/features/templates/templates.strings.js @@ -154,7 +154,9 @@ function TemplatesStrings (BaseString) { TIMEOUT: t.s('Timeout'), APPROVED: t.s('APPROVED'), DENIED: t.s('DENIED'), - ALL_PARENTS_MUST_CONVERGE: t.s('All parents must converge before running'), + CONVERGENCE: t.s('Convergence'), + ALL: t.s('All'), + ANY: t.s('Any'), }; } diff --git a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.controller.js b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.controller.js index 64bce33067..df75b083dd 100644 --- a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.controller.js +++ b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.controller.js @@ -106,6 +106,10 @@ export default ['$scope', 'TemplatesService', 'JobTemplateModel', 'PromptService element: '#workflow_node_edge', multiple: false }); + CreateSelect2({ + element: '#workflow_node_convergence', + multiple: false + }); }; const formatPopOverDetails = (model) => { @@ -500,10 +504,22 @@ export default ['$scope', 'TemplatesService', 'JobTemplateModel', 'PromptService type: 'workflow_job_template,job_template' }; - $scope.all_parents_must_converge = _.get( + const all_parents_must_converge = _.get( $scope, ['nodeConfig', 'node', 'all_parents_must_converge'], _.get($scope, ['nodeConfig', 'node', 'originalNodeObject', 'all_parents_must_converge'], false) ); + $scope.convergenceOptions = [ + { + label: $scope.strings.get('workflow_maker.ALL'), + value: true, + }, + { + label: $scope.strings.get('workflow_maker.ANY'), + value: false, + }, + ]; + $scope.convergenceChoice = $scope.convergenceOptions.find(({ value }) => value === all_parents_must_converge); + $scope.wf_maker_templates = []; $scope.wf_maker_template_dataset = {}; @@ -622,7 +638,7 @@ export default ['$scope', 'TemplatesService', 'JobTemplateModel', 'PromptService $scope.confirmNodeForm = () => { const nodeFormData = { edgeType: $scope.edgeType, - all_parents_must_converge: $scope.all_parents_must_converge, + all_parents_must_converge: $scope.convergenceChoice.value, }; if ($scope.activeTab === "approval") { diff --git a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html index cdc6c966aa..a8beac4418 100644 --- a/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html +++ b/awx/ui/client/src/templates/workflows/workflow-maker/forms/workflow-node-form.partial.html @@ -183,26 +183,22 @@
-
-
-
+