AAP-36536 Send job_lifecycle logs to external loggers (#15701)

* Send job_lifecycle logs to external loggers

* Include structured data in message

* Attach the organization_id of the job

awx/main/conf.py

@@ -592,7 +592,7 @@ register(
 register(
     'LOG_AGGREGATOR_LOGGERS',
     field_class=fields.StringListField,
-    default=['awx', 'activity_stream', 'job_events', 'system_tracking', 'broadcast_websocket'],
+    default=['awx', 'activity_stream', 'job_events', 'system_tracking', 'broadcast_websocket', 'job_lifecycle'],
     label=_('Loggers Sending Data to Log Aggregator Form'),
     help_text=_(
         'List of loggers that will send HTTP logs to the collector, these can '
@@ -602,6 +602,7 @@ register(
         'job_events - callback data from Ansible job events\n'
         'system_tracking - facts gathered from scan jobs\n'
         'broadcast_websocket - errors pertaining to websockets broadcast metrics\n'
+        'job_lifecycle - logs related to processing of a job\n'
     ),
     category=_('Logging'),
     category_slug='logging',

awx/main/models/unified_jobs.py

@@ -1625,7 +1625,7 @@ class UnifiedJob(
         elif state == "execution_node_chosen":
             extra["execution_node"] = self.execution_node or "NOT_SET"
 
-        logger_job_lifecycle.info(f"{msg} {json.dumps(extra)}")
+        logger_job_lifecycle.info(f"{msg} {json.dumps(extra)}", extra={'lifecycle_data': extra, 'organization_id': self.organization_id})
 
     @property
     def launched_by(self):

awx/main/utils/formatters.py

@@ -160,6 +160,11 @@ class LogstashFormatter(LogstashFormatterBase):
             data = json.loads(data)
         data_for_log = {}
 
+        # For the job_lifecycle logger, copy some raw data fields directly
+        for key in ('lifecycle_data', 'organization_id'):
+            if key in raw_data:
+                data_for_log[key] = raw_data[key]
+
         if kind == 'job_events' and raw_data.get('python_objects', {}).get('job_event'):
             job_event = raw_data['python_objects']['job_event']
             guid = job_event.event_data.pop('guid', None)

awx/settings/defaults.py

@@ -783,7 +783,7 @@ LOGGING = {
         'awx.analytics': {'handlers': ['external_logger'], 'level': 'INFO', 'propagate': False},
         'awx.analytics.broadcast_websocket': {'handlers': ['console', 'file', 'wsrelay', 'external_logger'], 'level': 'INFO', 'propagate': False},
         'awx.analytics.performance': {'handlers': ['console', 'file', 'tower_warnings', 'external_logger'], 'level': 'DEBUG', 'propagate': False},
-        'awx.analytics.job_lifecycle': {'handlers': ['console', 'job_lifecycle'], 'level': 'DEBUG', 'propagate': False},
+        'awx.analytics.job_lifecycle': {'handlers': ['console', 'job_lifecycle', 'external_logger'], 'level': 'DEBUG', 'propagate': False},
         'social': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'DEBUG'},
         'system_tracking_migrations': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'DEBUG'},
         'rbac_migrations': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'DEBUG'},

tools/docker-compose/docs/logstash.md

@@ -22,7 +22,8 @@ authentication set up inside of the logstash configuration file).
         "awx",
         "activity_stream",
         "job_events",
-        "system_tracking"
+        "system_tracking",
+        "job_lifecycle"
     ],
     "LOG_AGGREGATOR_INDIVIDUAL_FACTS": false,
     "LOG_AGGREGATOR_TOWER_UUID": "991ac7e9-6d68-48c8-bbde-7ca1096653c6",