Merge pull request #12391 from AlanCoding/compose_graphs

Do the grafana thing in docker-compose templating itself

.gitignore

@@ -38,7 +38,6 @@ awx/ui/build
 awx/ui/.env.local
 awx/ui/instrumented
 rsyslog.pid
-tools/prometheus
 tools/docker-compose/ansible/awx_dump.sql
 tools/docker-compose/Dockerfile
 tools/docker-compose/_build
@@ -154,6 +153,9 @@ use_dev_supervisor.txt
 /sanity/
 /awx_collection_build/
 
+# Setup for metrics gathering
+tools/prometheus/prometheus.yml
+
 .idea/*
 *.unison.tmp
 *.#

Makefile

@@ -17,6 +17,10 @@ KEYCLOAK ?= false
 LDAP ?= false
 # If set to true docker-compose will also start a splunk instance
 SPLUNK ?= false
+# If set to true docker-compose will also start a prometheus instance
+PROMETHEUS ?= false
+# If set to true docker-compose will also start a grafana instance
+GRAFANA ?= false
 
 VENV_BASE ?= /var/lib/awx/venv
 
@@ -470,7 +474,9 @@ docker-compose-sources: .git/hooks/pre-commit
 	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP) \
 	    -e enable_keycloak=$(KEYCLOAK) \
 	    -e enable_ldap=$(LDAP) \
-	    -e enable_splunk=$(SPLUNK)
+	    -e enable_splunk=$(SPLUNK) \
+	    -e enable_prometheus=$(PROMETHEUS) \
+	    -e enable_grafana=$(GRAFANA)
 
 
 docker-compose: awx/projects docker-compose-sources
@@ -518,7 +524,7 @@ docker-clean:
 	fi
 
 docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
-	docker volume rm tools_awx_db
+	docker volume rm -f tools_awx_db tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q)
 
 docker-refresh: docker-clean docker-compose
 
@@ -529,14 +535,6 @@ docker-compose-elk: awx/projects docker-compose-sources
 docker-compose-cluster-elk: awx/projects docker-compose-sources
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
-prometheus:
-	docker volume create prometheus
-	docker run -d --rm --net=_sources_default --link=awx_1:awx1 --volume prometheus-storage:/prometheus --volume `pwd`/tools/prometheus:/etc/prometheus --name prometheus -p 9090:9090 prom/prometheus
-
-grafana:
-	docker volume create grafana
-	docker run -d --rm --net=_sources_default --volume grafana-storage:/var/lib/grafana --volume `pwd`/tools/grafana:/etc/grafana/provisioning --name grafana -p 3001:3000 grafana/grafana-enterprise
-
 docker-compose-container-group:
 	MINIKUBE_CONTAINER_GROUP=true make docker-compose
 

tools/docker-compose/README.md

@@ -404,7 +404,7 @@ Anytime you want to run an OpenLDAP instance alongside AWX we can start docker-c
 LDAP=true make docker-compose
 ```
 
-Once the containers come up two new ports (389, 636) should be exposed and the LDAP server should be running on those ports. The first port (389) is non-SSL and the second port (636) is SSL enabled. 
+Once the containers come up two new ports (389, 636) should be exposed and the LDAP server should be running on those ports. The first port (389) is non-SSL and the second port (636) is SSL enabled.
 
 Now we are ready to configure and plumb OpenLDAP with AWX. To do this we have provided a playbook which will:
 * Backup and configure the LDAP adapter in AWX. NOTE: this will back up your existing settings but the password fields can not be backed up through the API, you need a DB backup to recover this.
@@ -427,7 +427,7 @@ Once the playbook is done running LDAP should now be setup in your development e
 3. awx_ldap_auditor:audit123
 4. awx_ldap_org_admin:orgadmin123
 
-The first account is a normal user. The second account will be a super user in AWX. The third account will be a system auditor in AWX. The fourth account is an org admin. All users belong to an org called "LDAP Organization". To log in with one of these users go to the AWX login screen enter the username/password. 
+The first account is a normal user. The second account will be a super user in AWX. The third account will be a system auditor in AWX. The fourth account is an org admin. All users belong to an org called "LDAP Organization". To log in with one of these users go to the AWX login screen enter the username/password.
 
 
 ### Splunk Integration
@@ -449,7 +449,7 @@ Once the containers are up we are ready to configure and plumb Splunk with AWX.
 * Backup and configure the External Logging adapter in AWX. NOTE: this will back up your existing settings but the password fields can not be backed up through the API, you need a DB backup to recover this.
 * Create a TCP port in Splunk for log forwarding
 
-For routing traffic between AWX and Splunk we will use the internal docker compose network. The `Logging Aggregator` will be configured using the internal network machine name of `splunk`. 
+For routing traffic between AWX and Splunk we will use the internal docker compose network. The `Logging Aggregator` will be configured using the internal network machine name of `splunk`.
 
 Once you have have the collections installed (from above) you can run the playbook like:
 ```bash
@@ -465,14 +465,26 @@ Once the playbook is done running Splunk should now be setup in your development
 
 Prometheus is a metrics collecting tool, and we support prometheus formatted data at the `api/v2/metrics` endpoint.
 
-1. Change the `username` and `password` in `tools/prometheus/prometheus.yml`. You can also change the scrape interval.
+Before you run anything, you should perform this basic setup:
+
+1. Copy the prometheus configuration:
+
+```
+cp tools/prometheus/prometheus.yml.example tools/prometheus/prometheus.yml
+```
+
+Set the `username` and `password` in that file to your AWX user. You can also change the scrape interval.
+
 2. (optional) if you are in a clustered environment, you can change the target to `haproxy:8043` so that the incoming prometheus requests go through the load balancer. Leaving it set to `awx1` also works.
-3. run `make prometheus`
-4. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up.
-5. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button)
 
-Prometheus can display basic graphs of your data, but it is minimal. Often Prometheus is paired with an app like Grafana for better visualization features.
+You can use this as part of the docker-compose target:
 
-1. `make grafana` to run a local docker grafana instance.
-2. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password.
-4. Now you can create a dashboard and add panels for whichever metrics you like.
+```
+PROMETHEUS=true GRAFANA=true make docker-compose
+```
+
+3. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up.
+4. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button)
+5. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password.
+6. In the left navigation menu go to Dashboards->Browse, find the "awx-demo" and click. These should have graphs.
+6. Now you can modify these and add panels for whichever metrics you like.

tools/docker-compose/ansible/roles/sources/defaults/main.yml

@@ -29,3 +29,5 @@ ldap_private_key_file: '{{ ldap_cert_dir }}/{{ ldap_private_key_file_name }}'
 ldap_cert_subject: "/C=US/ST=NC/L=Durham/O=awx/CN="
 
 enable_splunk: false
+enable_grafana: false
+enable_prometheus: false

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -135,6 +135,37 @@ services:
     environment:
       SPLUNK_START_ARGS: --accept-license
       SPLUNK_PASSWORD: splunk_admin
+{% endif %}
+{% if enable_prometheus|bool %}
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: tools_prometheus_1
+    hostname: splunk
+    ports:
+      - "9090:9090"
+    volumes:
+      - "../../prometheus:/etc/prometheus"
+      - "prometheus_storage:/prometheus:rw"
+    links:
+      - awx_1:awx1
+{% endif %}
+{% if enable_grafana|bool %}
+  grafana:
+    image: grafana/grafana-enterprise:latest
+    container_name: tools_grafana_1
+    hostname: splunk
+    ports:
+      - "3001:3000"
+    volumes:
+      - "../../grafana:/etc/grafana/provisioning"
+      - "grafana_storage:/var/lib/grafana:rw"
+    environment:
+      SPLUNK_START_ARGS: --accept-license
+      SPLUNK_PASSWORD: splunk_admin
+    links:
+      - prometheus
+    depends_on:
+      - prometheus
 {% endif %}
   # A useful container that simply passes through log messages to the console
   # helpful for testing awx/tower logging
@@ -198,6 +229,14 @@ volumes:
     name: tools_ldap_1
     driver: local
 {% endif %}
+{% if enable_prometheus|bool %}
+  prometheus_storage:
+    name: tools_prometheus_storage
+{% endif %}
+{% if enable_grafana|bool %}
+  grafana_storage:
+    name: tools_grafana_storage
+{% endif %}
 {% if minikube_container_group|bool %}
 networks:
   default:

tools/prometheus/prometheus.yml.example

@@ -14,5 +14,5 @@ scrape_configs:
     params:
       format: ['txt']
     basic_auth:
-      username: awxuser  # change this
+      username: admin  # change this
       password: password  # change this