drastically improved performance by removing unnecessary iteration over children of parent nodes, additionally added an extra check that the node didn't already have a job so that it wasn't cycling over nodes that had already run when running through all_nodes

This commit is contained in:
Rebeccah
2020-01-22 17:47:38 -05:00
parent 82dd4a3884
commit 6d2a2ab714

View File

@@ -210,34 +210,30 @@ class WorkflowDAG(SimpleDAG):
for node in self.sort_nodes_topological(): for node in self.sort_nodes_topological():
obj = node['node_object'] obj = node['node_object']
if obj.do_not_run is False and not obj.job and node not in root_nodes and not obj.all_parents_must_converge:
parent_nodes = [p['node_object'] for p in self.get_parents(obj)] parent_nodes = [p['node_object'] for p in self.get_parents(obj)]
if obj.do_not_run is False and not obj.job and node not in root_nodes and not obj.all_parents_must_converge:
if self._are_all_nodes_dnr_decided(parent_nodes): if self._are_all_nodes_dnr_decided(parent_nodes):
if self._should_mark_node_dnr(node, parent_nodes): if self._should_mark_node_dnr(node, parent_nodes):
obj.do_not_run = True obj.do_not_run = True
nodes_marked_do_not_run.append(node) nodes_marked_do_not_run.append(node)
if obj.do_not_run is False and obj.all_parents_must_converge: if obj.do_not_run is False and obj.all_parents_must_converge and not obj.job:
if self._are_all_nodes_dnr_decided(parent_nodes):
if self._are_relevant_parents_finished(node): if self._are_relevant_parents_finished(node):
# if the current node is a convergence node and all the # if the current node is a convergence node and all the
# parents are finished then check to see if all parents # parents are finished then check to see if all parents
# met the needed criteria to run the convergence child # met the needed criteria to run the convergence child
# (i.e. parent must fail, parent must succeed) # (i.e. parent must fail, parent must succeed)
parent_nodes = [p['node_object'] for p in self.get_parents(obj)]
if any(p.do_not_run for p in parent_nodes): if any(p.do_not_run for p in parent_nodes):
obj.do_not_run = True obj.do_not_run = True
nodes_marked_do_not_run.append(node) nodes_marked_do_not_run.append(node)
else: else:
# import sdb
# sdb.set_trace()
for p in parent_nodes: for p in parent_nodes:
child_nodes = []
if p.job.status == "successful": if p.job.status == "successful":
child_nodes = [x for x in self.get_children(p, "success_nodes")] status = "success_nodes"
elif p.job.status == "failed": elif p.job.status == "failed":
child_nodes = [x for x in self.get_children(p, "failure_nodes")] status = "failure_nodes"
child_nodes.extend(x for x in self.get_children(p, "always_nodes")) if (p not in [node['node_object'] for node in self.get_parents(obj, status)]
if node not in child_nodes: and p not in [node['node_object'] for node in self.get_parents(obj, "always_nodes")]):
obj.do_not_run = True obj.do_not_run = True
nodes_marked_do_not_run.append(node) nodes_marked_do_not_run.append(node)
break break