Add subsystem metrics

- Adds a Metrics() class that can track data such as number of
events the callback receiver inserted into database
- Exposes this metric data at the api/v2/metrics/ endpoint.
This data is prometheus-friendly
- Metric data is stored in memory, then periodically saved to Redis.
- Metric data is periodically broadcast to other nodes in the cluster,
so that each node has a copy of the most recent metric data collected.

awx/api/renderers.py

@@ -129,6 +129,18 @@ class PrometheusJSONRenderer(renderers.JSONRenderer):
         parsed_metrics = text_string_to_metric_families(data)
         data = {}
         for family in parsed_metrics:
+            data[family.name] = {}
+            data[family.name]['help_text'] = family.documentation
+            data[family.name]['type'] = family.type
+            data[family.name]['samples'] = []
             for sample in family.samples:
-                data[sample[0]] = {"labels": sample[1], "value": sample[2]}
+                sample_dict = {"labels": sample[1], "value": sample[2]}
+                if family.type == 'histogram':
+                    if sample[0].endswith("_sum"):
+                        sample_dict['sample_type'] = "sum"
+                    elif sample[0].endswith("_count"):
+                        sample_dict['sample_type'] = "count"
+                    elif sample[0].endswith("_bucket"):
+                        sample_dict['sample_type'] = "bucket"
+                data[family.name]['samples'].append(sample_dict)
         return super(PrometheusJSONRenderer, self).render(data, accepted_media_type, renderer_context)

awx/api/templates/api/metrics_view.md

@@ -0,0 +1 @@
+query params to filter response, e.g., ?subsystemonly=1&metric=callback_receiver_events_insert_db&node=awx-1

awx/api/views/metrics.py

@@ -14,6 +14,7 @@ from rest_framework.exceptions import PermissionDenied
 
 # AWX
 # from awx.main.analytics import collectors
+import awx.main.analytics.subsystem_metrics as s_metrics
 from awx.main.analytics.metrics import metrics
 from awx.api import renderers
 
@@ -33,5 +34,10 @@ class MetricsView(APIView):
     def get(self, request):
         ''' Show Metrics Details '''
         if request.user.is_superuser or request.user.is_system_auditor:
-            return Response(metrics().decode('UTF-8'))
+            metrics_to_show = ''
+            if not request.query_params.get('subsystemonly', "0") == "1":
+                metrics_to_show += metrics().decode('UTF-8')
+            if not request.query_params.get('dbonly', "0") == "1":
+                metrics_to_show += s_metrics.metrics(request)
+            return Response(metrics_to_show)
         raise PermissionDenied()