Add timeout for workflow approval nodes

awx/main/migrations/0083_v360_workflow_approval.py

@@ -1,4 +1,4 @@
-# Generated by Django 2.2.2 on 2019-07-25 19:16
+# Generated by Django 2.2.4 on 2019-08-02 17:51
 
 import awx.main.fields
 from django.db import migrations, models

awx/main/models/workflow.py

@@ -661,6 +661,9 @@ class WorkflowApproval(UnifiedJob):
     def event_class(self):
         return None
 
+    def _get_parent_field_name(self):
+        return 'workflow_approval_template'
+
     def approve(self, request=None):
         self.status = 'successful'
         self.save()

awx/main/scheduler/task_manager.py

@@ -23,6 +23,7 @@ from awx.main.models import (
     Project,
     ProjectUpdate,
     SystemJob,
+    WorkflowApproval,
     WorkflowJob,
     WorkflowJobTemplate
 )
@@ -518,6 +519,19 @@ class TaskManager():
             if not found_acceptable_queue:
                 logger.debug("{} couldn't be scheduled on graph, waiting for next cycle".format(task.log_format))
 
+    def timeout_approval_node(self):
+        workflow_approval = WorkflowApproval.objects.filter(status='pending').prefetch_related('workflow_approval_template')
+        now = tz_now()
+        for task in workflow_approval:
+            approval_timeout_seconds = timedelta(seconds=task.workflow_approval_template.timeout)
+            if task.workflow_approval_template.timeout == 0:
+                continue
+            if (now - task.created) >= approval_timeout_seconds:
+                logger.info("This approval node has timed out.")
+                task.status = 'failed'
+                task.job_explanation = _("This approval node has timed out.")
+                task.save(update_fields=['status', 'job_explanation'])
+
     def calculate_capacity_consumed(self, tasks):
         self.graph = InstanceGroup.objects.capacity_values(tasks=tasks, graph=self.graph)
 
@@ -573,6 +587,8 @@ class TaskManager():
 
             self.spawn_workflow_graph_jobs(running_workflow_tasks)
 
+            self.timeout_approval_node()
+
             self.process_tasks(all_sorted_tasks)
         return finished_wfjs