Merge pull request #13608 from AlanCoding/keepalive

Use ansible-runner change to get periodic keep-alive messages in K8S

awx/main/conf.py

@@ -282,6 +282,16 @@ register(
     placeholder={'HTTP_PROXY': 'myproxy.local:8080'},
 )
 
+register(
+    'AWX_RUNNER_KEEPALIVE_SECONDS',
+    field_class=fields.IntegerField,
+    label=_('K8S Ansible Runner Keep-Alive Message Interval'),
+    help_text=_('Only applies to jobs running in a Container Group. If not 0, send a message every so-many seconds to keep connection open.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    placeholder=240,  # intended to be under common 5 minute idle timeout
+)
+
 register(
     'GALAXY_TASK_ENV',
     field_class=fields.KeyValueField,

awx/main/tasks/callback.py

@@ -85,6 +85,8 @@ class RunnerCallback:
         # which generate job events from two 'streams':
         # ansible-inventory and the awx.main.commands.inventory_import
         # logger
+        if event_data.get('event') == 'keepalive':
+            return
 
         if event_data.get(self.event_data_key, None):
             if self.event_data_key != 'job_id':

awx/main/tasks/receptor.py

@@ -526,6 +526,10 @@ class AWXReceptorJob:
         pod_spec['spec']['containers'][0]['image'] = ee.image
         pod_spec['spec']['containers'][0]['args'] = ['ansible-runner', 'worker', '--private-data-dir=/runner']
 
+        if settings.AWX_RUNNER_KEEPALIVE_SECONDS:
+            pod_spec['spec']['containers'][0].setdefault('env', [])
+            pod_spec['spec']['containers'][0]['env'].append({'name': 'ANSIBLE_RUNNER_KEEPALIVE_SECONDS', 'value': str(settings.AWX_RUNNER_KEEPALIVE_SECONDS)})
+
         # Enforce EE Pull Policy
         pull_options = {"always": "Always", "missing": "IfNotPresent", "never": "Never"}
         if self.task and self.task.instance.execution_environment:

awx/settings/defaults.py

@@ -929,6 +929,11 @@ AWX_RUNNER_OMIT_ENV_FILES = True
 # Allow ansible-runner to save ansible output (may cause performance issues)
 AWX_RUNNER_SUPPRESS_OUTPUT_FILE = True
 
+# https://github.com/ansible/ansible-runner/pull/1191/files
+# Interval in seconds between the last message and keep-alive messages that
+# ansible-runner will send
+AWX_RUNNER_KEEPALIVE_SECONDS = 0
+
 # Delete completed work units in receptor
 RECEPTOR_RELEASE_WORK = True
 

awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.js

@@ -150,6 +150,11 @@ function JobsEdit() {
                   type={options?.SCHEDULE_MAX_JOBS ? 'number' : undefined}
                   isRequired={Boolean(options?.SCHEDULE_MAX_JOBS)}
                 />
+                <InputField
+                  name="AWX_RUNNER_KEEPALIVE_SECONDS"
+                  config={jobs.AWX_RUNNER_KEEPALIVE_SECONDS}
+                  type="number"
+                />
                 <InputField
                   name="DEFAULT_JOB_TIMEOUT"
                   config={jobs.DEFAULT_JOB_TIMEOUT}

awx/ui/src/screens/Setting/shared/data.allSettingOptions.json

@@ -344,6 +344,16 @@
         "category_slug": "jobs",
         "default": 10
       },
+      "AWX_RUNNER_KEEPALIVE_SECONDS": {
+        "type": "integer",
+        "required": true,
+        "label": "K8S Ansible Runner Keep-Alive Message Interval",
+        "help_text": "Only applies to K8S deployments and container_group jobs. If not 0, send a message every so-many seconds to keep connection open.",
+        "category": "Jobs",
+        "category_slug": "jobs",
+        "placeholder": 240,
+        "default": 0
+      },
       "AWX_ANSIBLE_CALLBACK_PLUGINS": {
         "type": "list",
         "required": false,
@@ -4098,6 +4108,15 @@
         "category_slug": "jobs",
         "defined_in_file": false
       },
+      "AWX_RUNNER_KEEPALIVE_SECONDS": {
+        "type": "integer",
+        "label": "K8S Ansible Runner Keep-Alive Message Interval",
+        "help_text": "Only applies to K8S deployments and container_group jobs. If not 0, send a message every so-many seconds to keep connection open.",
+        "category": "Jobs",
+        "category_slug": "jobs",
+        "placeholder": 240,
+        "default": 0
+      },
       "AWX_ANSIBLE_CALLBACK_PLUGINS": {
         "type": "list",
         "label": "Ansible Callback Plugins",

awx/ui/src/screens/Setting/shared/data.allSettings.json

@@ -51,6 +51,7 @@
   "STDOUT_MAX_BYTES_DISPLAY":1048576,
   "EVENT_STDOUT_MAX_BYTES_DISPLAY":1024,
   "SCHEDULE_MAX_JOBS":10,
+  "AWX_RUNNER_KEEPALIVE_SECONDS": 0,
   "AWX_ANSIBLE_CALLBACK_PLUGINS":[],
   "DEFAULT_JOB_TIMEOUT":0,
   "DEFAULT_JOB_IDLE_TIMEOUT":0,

awx/ui/src/screens/Setting/shared/data.jobSettings.json

@@ -19,6 +19,7 @@
   "STDOUT_MAX_BYTES_DISPLAY": 1048576,
   "EVENT_STDOUT_MAX_BYTES_DISPLAY": 1024,
   "SCHEDULE_MAX_JOBS": 10,
+  "AWX_RUNNER_KEEPALIVE_SECONDS": 0,
   "AWX_ANSIBLE_CALLBACK_PLUGINS": [],
   "DEFAULT_JOB_TIMEOUT": 0,
   "DEFAULT_JOB_IDLE_TIMEOUT": 0,