Merge pull request #3636 from ansible/prometheus

Prometheus Metrics

Reviewed-by: https://github.com/softwarefactory-project-zuul[bot]

Makefile

@@ -631,6 +631,9 @@ docker-compose-elk: docker-auth
 docker-compose-cluster-elk: docker-auth
 	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
+prometheus:
+	docker run -u0 --net=tools_default --link=`docker ps | egrep -o "tools_awx(_run)?_([^ ]+)?"`:awxweb --volume `pwd`/tools/prometheus:/prometheus --name prometheus -d -p 0.0.0.0:9090:9090 prom/prometheus --web.enable-lifecycle --config.file=/prometheus/prometheus.yml
+
 minishift-dev:
 	ansible-playbook -i localhost, -e devtree_directory=$(CURDIR) tools/clusterdevel/start_minishift_dev.yml
 

awx/api/metrics.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    MetricsView
+)
+
+
+urls = [
+    url(r'^$', MetricsView.as_view(), name='metrics_view'),
+]
+
+__all__ = ['urls']

awx/api/urls/urls.py

@@ -34,6 +34,8 @@ from awx.api.views import (
     OAuth2ApplicationDetail,
 )
 
+from awx.api.views.metrics import MetricsView
+
 from .organization import urls as organization_urls
 from .user import urls as user_urls
 from .project import urls as project_urls
@@ -133,6 +135,7 @@ v2_urls = [
     url(r'^applications/(?P<pk>[0-9]+)/tokens/$', ApplicationOAuth2TokenList.as_view(), name='application_o_auth2_token_list'),
     url(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'),
     url(r'^', include(oauth2_urls)),
+    url(r'^metrics/$', MetricsView.as_view(), name='metrics_view'),
 ]
 
 app_name = 'api'

awx/api/views/metrics.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+
+# Python
+import logging
+
+# Django
+from django.utils.translation import ugettext_lazy as _
+
+# Django REST Framework
+from rest_framework.response import Response
+from rest_framework.renderers import JSONRenderer
+from rest_framework.exceptions import PermissionDenied
+
+
+# AWX
+# from awx.main.analytics import collectors
+from awx.main.analytics.metrics import metrics
+from awx.api import renderers
+
+from awx.api.generics import (
+    APIView,
+)
+
+
+logger = logging.getLogger('awx.main.analytics')
+
+
+class MetricsView(APIView):
+
+    view_name = _('Metrics')
+    swagger_topic = 'Metrics'
+
+    renderer_classes = [renderers.PlainTextRenderer,
+                        renderers.BrowsableAPIRenderer,
+                        JSONRenderer,]
+
+    def get(self, request, format='txt'):
+        ''' Show Metrics Details '''
+        if (request.user.is_superuser or request.user.is_system_auditor):
+            return Response(metrics().decode('UTF-8'))
+        raise PermissionDenied()

awx/api/views/root.py

@@ -104,6 +104,7 @@ class ApiVersionRootView(APIView):
             data['credential_input_sources'] = reverse('api:credential_input_source_list', request=request)
             data['applications'] = reverse('api:o_auth2_application_list', request=request)
             data['tokens'] = reverse('api:o_auth2_token_list', request=request)
+            data['metrics'] = reverse('api:metrics_view', request=request)
         data['inventory'] = reverse('api:inventory_list', request=request)
         data['inventory_scripts'] = reverse('api:inventory_script_list', request=request)
         data['inventory_sources'] = reverse('api:inventory_source_list', request=request)
@@ -278,6 +279,3 @@ class ApiV1ConfigView(APIView):
         except Exception:
             # FIX: Log
             return Response({"error": _("Failed to remove license.")}, status=status.HTTP_400_BAD_REQUEST)
-
-
-

awx/main/analytics/collectors.py

@@ -159,15 +159,17 @@ def instance_info(since):
     instances = models.Instance.objects.values_list('hostname').annotate().values(
         'uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'hostname', 'last_isolated_check', 'enabled')
     for instance in instances:
-        info = {'uuid': instance['uuid'],
+        instance_info = {
-                'version': instance['version'],
+            'uuid': instance['uuid'],
-                'capacity': instance['capacity'],
+            'version': instance['version'],
-                'cpu': instance['cpu'],
+            'capacity': instance['capacity'],
-                'memory': instance['memory'],
+            'cpu': instance['cpu'],
-                'managed_by_policy': instance['managed_by_policy'],
+            'memory': instance['memory'],
-                'last_isolated_check': instance['last_isolated_check'],
+            'managed_by_policy': instance['managed_by_policy'],
-                'enabled': instance['enabled']
+            'last_isolated_check': instance['last_isolated_check'],
-                }
+            'enabled': instance['enabled']
+        }
+        info[instance['uuid']] = instance_info
     return info
 
 
@@ -187,12 +189,12 @@ def job_instance_counts(since):
     job_types = models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
         'execution_node', 'launch_type').annotate(job_launch_type=Count('launch_type'))
     for job in job_types:
-        counts.setdefault(job[0], {}).setdefault('status', {})[job[1]] = job[2]
+        counts.setdefault(job[0], {}).setdefault('launch_type', {})[job[1]] = job[2]
         
     job_statuses = models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
         'execution_node', 'status').annotate(job_status=Count('status'))
     for job in job_statuses:
-        counts.setdefault(job[0], {}).setdefault('launch_type', {})[job[1]] = job[2]
+        counts.setdefault(job[0], {}).setdefault('status', {})[job[1]] = job[2]
     return counts
 
 

awx/main/analytics/metrics.py

@@ -0,0 +1,118 @@
+from datetime import datetime
+from django.conf import settings
+from prometheus_client import (
+    REGISTRY,
+    PROCESS_COLLECTOR,
+    PLATFORM_COLLECTOR,
+    GC_COLLECTOR,
+    Gauge,
+    Info,
+    generate_latest
+)
+
+from awx.conf.license import get_license
+from awx.main.utils import (get_awx_version, get_ansible_version)
+from awx.main.analytics.collectors import (
+    counts, 
+    instance_info,
+    job_instance_counts,
+)
+
+
+REGISTRY.unregister(PROCESS_COLLECTOR)
+REGISTRY.unregister(PLATFORM_COLLECTOR)
+REGISTRY.unregister(GC_COLLECTOR)
+
+SYSTEM_INFO = Info('awx_system', 'AWX System Information')
+ORG_COUNT = Gauge('awx_organizations_total', 'Number of organizations')
+USER_COUNT = Gauge('awx_users_total', 'Number of users')
+TEAM_COUNT = Gauge('awx_teams_total', 'Number of teams')
+INV_COUNT = Gauge('awx_inventories_total', 'Number of inventories')
+PROJ_COUNT = Gauge('awx_projects_total', 'Number of projects')
+JT_COUNT = Gauge('awx_job_templates_total', 'Number of job templates')
+WFJT_COUNT = Gauge('awx_workflow_job_templates_total', 'Number of workflow job templates')
+HOST_COUNT = Gauge('awx_hosts_total', 'Number of hosts', ['type',])
+SCHEDULE_COUNT = Gauge('awx_schedules_total', 'Number of schedules')
+INV_SCRIPT_COUNT = Gauge('awx_inventory_scripts_total', 'Number of invetory scripts')
+USER_SESSIONS = Gauge('awx_sessions_total', 'Number of sessions', ['type',])
+CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs')
+RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the Tower system')
+
+INSTANCE_CAPACITY = Gauge('awx_instance_capacity', 'Capacity of each node in a Tower system', ['type',])
+INSTANCE_CPU = Gauge('awx_instance_cpu', 'CPU cores on each node in a Tower system', ['type',])
+INSTANCE_MEMORY = Gauge('awx_instance_memory', 'RAM (Kb) on each node in a Tower system', ['type',])
+INSTANCE_INFO = Info('awx_instance', 'Info about each node in a Tower system', ['type',])
+INSTANCE_LAUNCH_TYPE = Gauge('awx_instance_launch_type_total', 'Type of Job launched', ['node', 'launch_type',])
+INSTANCE_STATUS = Gauge('awx_instance_status_total', 'Status of Job launched', ['node', 'status',])
+
+
+def metrics():
+    license_info = get_license(show_key=False)
+    SYSTEM_INFO.info({
+        'system_uuid': settings.SYSTEM_UUID,
+        'insights_analytics': str(settings.INSIGHTS_DATA_ENABLED),
+        'tower_url_base': settings.TOWER_URL_BASE,
+        'tower_version': get_awx_version(),
+        'ansible_version': get_ansible_version(),
+        'license_type': license_info.get('license_type', 'UNLICENSED'),
+        'free_instances': str(license_info.get('free instances', 0)),
+        'license_expiry': str(license_info.get('time_remaining', 0)),
+        'pendo_tracking': settings.PENDO_TRACKING_STATE,
+        'external_logger_enabled': str(settings.LOG_AGGREGATOR_ENABLED),
+        'external_logger_type': getattr(settings, 'LOG_AGGREGATOR_TYPE', 'None')
+    })
+
+    current_counts = counts(datetime.now())
+
+    ORG_COUNT.set(current_counts['organization'])
+    USER_COUNT.set(current_counts['user'])
+    TEAM_COUNT.set(current_counts['team'])
+    INV_COUNT.set(current_counts['inventory'])
+    PROJ_COUNT.set(current_counts['project'])
+    JT_COUNT.set(current_counts['job_template'])
+    WFJT_COUNT.set(current_counts['workflow_job_template'])
+
+    HOST_COUNT.labels(type='all').set(current_counts['host'])
+    HOST_COUNT.labels(type='active').set(current_counts['active_host_count'])
+
+    SCHEDULE_COUNT.set(current_counts['schedule'])
+    INV_SCRIPT_COUNT.set(current_counts['custom_inventory_script'])
+    CUSTOM_VENVS.set(current_counts['custom_virtualenvs'])
+
+    USER_SESSIONS.labels(type='all').set(current_counts['active_sessions'])
+    USER_SESSIONS.labels(type='user').set(current_counts['active_user_sessions'])
+    USER_SESSIONS.labels(type='anonymous').set(current_counts['active_anonymous_sessions'])
+
+    RUNNING_JOBS.set(current_counts['running_jobs'])
+
+
+    instance_data = instance_info(datetime.now())
+    for uuid in instance_data:
+        INSTANCE_CAPACITY.labels(type=uuid).set(instance_data[uuid]['capacity'])
+        INSTANCE_CPU.labels(type=uuid).set(instance_data[uuid]['cpu'])
+        INSTANCE_MEMORY.labels(type=uuid).set(instance_data[uuid]['memory'])
+        INSTANCE_INFO.labels(type=uuid).info({
+            'enabled': str(instance_data[uuid]['enabled']),
+            'last_isolated_check': getattr(instance_data[uuid], 'last_isolated_check', 'None'),
+            'managed_by_policy': str(instance_data[uuid]['managed_by_policy']),
+            'version': instance_data[uuid]['version']
+        })
+
+    instance_data = job_instance_counts(datetime.now())
+    for node in instance_data:
+        # skipping internal execution node (for system jobs) 
+        # TODO: determine if we should exclude execution_node from instance count
+        if node == '':
+            continue
+        types = instance_data[node].get('launch_type', {})
+        for launch_type, value in types.items():
+            INSTANCE_LAUNCH_TYPE.labels(node=node, launch_type=launch_type).set(value)
+        statuses = instance_data[node].get('status', {})
+        for status, value in statuses.items():
+            INSTANCE_STATUS.labels(node=node, status=status).set(value)
+
+
+    return generate_latest()
+
+
+__all__ = ['metrics']

awx/main/tests/functional/analytics/test_metrics.py

@@ -0,0 +1,56 @@
+import pytest
+
+from prometheus_client.parser import text_string_to_metric_families
+from awx.main import models
+from awx.main.analytics.metrics import metrics
+
+EXPECTED_VALUES = {
+    'awx_system_info':1.0,
+    'awx_organizations_total':1.0,
+    'awx_users_total':1.0,
+    'awx_teams_total':1.0,
+    'awx_inventories_total':1.0,
+    'awx_projects_total':1.0,
+    'awx_job_templates_total':1.0,
+    'awx_workflow_job_templates_total':1.0,
+    'awx_hosts_total':1.0,
+    'awx_hosts_total':1.0,
+    'awx_schedules_total':1.0,
+    'awx_inventory_scripts_total':1.0,
+    'awx_sessions_total':0.0,
+    'awx_sessions_total':0.0,
+    'awx_sessions_total':0.0,
+    'awx_custom_virtualenvs_total':0.0,
+    'awx_running_jobs_total':0.0,
+    'awx_instance_capacity':100.0,
+    'awx_instance_cpu':0.0,
+    'awx_instance_memory':0.0,
+    'awx_instance_info':1.0,
+}
+
+
+@pytest.mark.django_db
+def test_metrics_counts(organization_factory, job_template_factory, workflow_job_template_factory):
+    objs = organization_factory('org', superusers=['admin'])
+    jt = job_template_factory(
+        'test', organization=objs.organization,
+        inventory='test_inv', project='test_project',
+        credential='test_cred'
+    )
+    workflow_job_template_factory('test')
+    models.Team(organization=objs.organization).save()
+    models.Host(inventory=jt.inventory).save()
+    models.Schedule(
+        rrule='DTSTART;TZID=America/New_York:20300504T150000',
+        unified_job_template=jt.job_template
+    ).save()
+    models.CustomInventoryScript(organization=objs.organization).save()
+
+    output = metrics()
+    gauges = text_string_to_metric_families(output.decode('UTF-8'))
+
+    for gauge in gauges:
+        for sample in gauge.samples:
+            # name, label, value, timestamp, exemplar
+            name, _, value, _, _ = sample
+            assert EXPECTED_VALUES[name] == value

docs/licenses/prometheus-client.txt

@@ -0,0 +1,202 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

docs/prometheus.md

@@ -0,0 +1,11 @@
+# Prometheus Support
+
+## Development
+AWX comes with an example prometheus container and make target.  To use it:
+
+1.  Edit `tools/prometheus/prometheus.yml` and update the `basic_auth` section
+    to specify a valid user/password for an AWX user you've created.
+    Alternatively, you can provide an OAuth2 token (which can be generated at
+    `/api/v2/users/N/personal_tokens/`).
+2.  Start the Prometheus container:
+    `make prometheus`

requirements/requirements.in

@@ -29,6 +29,7 @@ jsonschema==2.6.0
 Markdown==2.6.11    # used for formatting API help
 ordereddict==1.1
 pexpect==4.6.0
+prometheus_client==0.6.0
 psutil==5.4.3
 psycopg2==2.7.3.2   # problems with Segmentation faults / wheels on upgrade
 pygerduty==0.37.0

requirements/requirements.txt

@@ -74,6 +74,7 @@ oauthlib==2.0.6           # via django-oauth-toolkit, requests-oauthlib, social-
 ordereddict==1.1
 pexpect==4.6.0
 pkgconfig==1.4.0          # via xmlsec
+prometheus_client==0.6.0
 psutil==5.4.3
 psycopg2==2.7.3.2
 ptyprocess==0.6.0         # via pexpect

tools/prometheus/.gitignore

@@ -0,0 +1 @@
+./data

tools/prometheus/prometheus.yml

@@ -0,0 +1,45 @@
+# prometheus.yml
+# my global config
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+    # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+        # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+    # - "first_rules.yml"
+    # - "second_rules.yml"
+
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+      # metrics_path defaults to '/metrics'
+      # scheme defaults to 'http'.
+    static_configs:
+    - targets: ['127.0.0.1:9090']
+
+  - job_name: 'awx'
+    tls_config:
+      insecure_skip_verify: True
+    metrics_path: /api/v2/metrics
+    scrape_interval: 5s
+    scheme: http
+    params:
+      format: ['txt']
+    basic_auth:
+      username: admin
+      password: password
+    # bearer_token: oauth-token
+    static_configs:
+      - targets:
+        - awxweb:8013