From 11a6e98230194b3d2b6c73c6b57fb0f8a7f8844d Mon Sep 17 00:00:00 2001
From: Christian Adams <rooftopcellist@gmail.com>
Date: Thu, 25 Jul 2019 13:29:01 -0400
Subject: [PATCH] Add pending jobs and system level job status to metrics

---
 awx/main/analytics/collectors.py                    | 1 +
 awx/main/analytics/metrics.py                       | 9 +++++++++
 awx/main/tests/functional/analytics/test_counts.py  | 3 ++-
 awx/main/tests/functional/analytics/test_metrics.py | 1 +
 4 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py
index 1a271363d5..0e0062aabc 100644
--- a/awx/main/analytics/collectors.py
+++ b/awx/main/analytics/collectors.py
@@ -93,6 +93,7 @@ def counts(since):
     counts['active_user_sessions'] = active_user_sessions
     counts['active_anonymous_sessions'] = active_anonymous_sessions
     counts['running_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').filter(status__in=('running', 'waiting',)).count()
+    counts['pending_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').filter(status__in=('pending',)).count()
     return counts
 
     
diff --git a/awx/main/analytics/metrics.py b/awx/main/analytics/metrics.py
index 3413e4670b..a418f271db 100644
--- a/awx/main/analytics/metrics.py
+++ b/awx/main/analytics/metrics.py
@@ -15,6 +15,7 @@ from awx.main.analytics.collectors import (
     counts, 
     instance_info,
     job_instance_counts,
+    job_counts,
 )
 
 
@@ -36,6 +37,8 @@ INV_SCRIPT_COUNT = Gauge('awx_inventory_scripts_total', 'Number of invetory scri
 USER_SESSIONS = Gauge('awx_sessions_total', 'Number of sessions', ['type',])
 CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs')
 RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the Tower system')
+PENDING_JOBS = Gauge('awx_pending_jobs_total', 'Number of pending jobs on the Tower system')
+STATUS = Gauge('awx_status_total', 'Status of Job launched', ['status',])
 
 INSTANCE_CAPACITY = Gauge('awx_instance_capacity', 'Capacity of each node in a Tower system', ['hostname', 'instance_uuid',])
 INSTANCE_CPU = Gauge('awx_instance_cpu', 'CPU cores on each node in a Tower system', ['hostname', 'instance_uuid',])
@@ -87,7 +90,13 @@ def metrics():
     USER_SESSIONS.labels(type='user').set(current_counts['active_user_sessions'])
     USER_SESSIONS.labels(type='anonymous').set(current_counts['active_anonymous_sessions'])
 
+    all_job_data = job_counts(None)
+    statuses = all_job_data.get('status', {})
+    for status, value in statuses.items():
+        STATUS.labels(status=status).set(value)
+
     RUNNING_JOBS.set(current_counts['running_jobs'])
+    PENDING_JOBS.set(current_counts['pending_jobs'])
 
     instance_data = instance_info(None, include_hostnames=True)
     for uuid, info in instance_data.items():
diff --git a/awx/main/tests/functional/analytics/test_counts.py b/awx/main/tests/functional/analytics/test_counts.py
index 3e2c98df03..1c68b6fa34 100644
--- a/awx/main/tests/functional/analytics/test_counts.py
+++ b/awx/main/tests/functional/analytics/test_counts.py
@@ -26,7 +26,8 @@ def test_empty():
         "team": 0,
         "user": 0,
         "workflow_job_template": 0,
-        "unified_job": 0
+        "unified_job": 0,
+        "pending_jobs": 0
     }
 
 
diff --git a/awx/main/tests/functional/analytics/test_metrics.py b/awx/main/tests/functional/analytics/test_metrics.py
index 385c299aea..c4d7c517c7 100644
--- a/awx/main/tests/functional/analytics/test_metrics.py
+++ b/awx/main/tests/functional/analytics/test_metrics.py
@@ -30,6 +30,7 @@ EXPECTED_VALUES = {
     'awx_instance_info':1.0,
     'awx_license_instance_total':0,
     'awx_license_instance_free':0,
+    'awx_pending_jobs_total':0,
 }