Merge pull request #12391 from AlanCoding/compose_graphs

Do the grafana thing in docker-compose templating itself

.gitignore

@@ -38,7 +38,6 @@ awx/ui/build
 awx/ui/.env.local
 awx/ui/instrumented
 rsyslog.pid
-tools/prometheus
 tools/docker-compose/ansible/awx_dump.sql
 tools/docker-compose/Dockerfile
 tools/docker-compose/_build
@@ -154,6 +153,9 @@ use_dev_supervisor.txt
 /sanity/
 /awx_collection_build/
 
+# Setup for metrics gathering
+tools/prometheus/prometheus.yml
+
 .idea/*
 *.unison.tmp
 *.#

Makefile

@@ -17,6 +17,10 @@ KEYCLOAK ?= false
 LDAP ?= false
 # If set to true docker-compose will also start a splunk instance
 SPLUNK ?= false
+# If set to true docker-compose will also start a prometheus instance
+PROMETHEUS ?= false
+# If set to true docker-compose will also start a grafana instance
+GRAFANA ?= false
 
 VENV_BASE ?= /var/lib/awx/venv
 
@@ -470,7 +474,9 @@ docker-compose-sources: .git/hooks/pre-commit
 	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP) \
 	    -e enable_keycloak=$(KEYCLOAK) \
 	    -e enable_ldap=$(LDAP) \
-	    -e enable_splunk=$(SPLUNK)
+	    -e enable_splunk=$(SPLUNK) \
+	    -e enable_prometheus=$(PROMETHEUS) \
+	    -e enable_grafana=$(GRAFANA)
 
 
 docker-compose: awx/projects docker-compose-sources
@@ -518,7 +524,7 @@ docker-clean:
 	fi
 
 docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
-	docker volume rm tools_awx_db
+	docker volume rm -f tools_awx_db tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q)
 
 docker-refresh: docker-clean docker-compose
 
@@ -529,14 +535,6 @@ docker-compose-elk: awx/projects docker-compose-sources
 docker-compose-cluster-elk: awx/projects docker-compose-sources
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
-prometheus:
-	docker volume create prometheus
-	docker run -d --rm --net=_sources_default --link=awx_1:awx1 --volume prometheus-storage:/prometheus --volume `pwd`/tools/prometheus:/etc/prometheus --name prometheus -p 9090:9090 prom/prometheus
-
-grafana:
-	docker volume create grafana
-	docker run -d --rm --net=_sources_default --volume grafana-storage:/var/lib/grafana --volume `pwd`/tools/grafana:/etc/grafana/provisioning --name grafana -p 3001:3000 grafana/grafana-enterprise
-
 docker-compose-container-group:
 	MINIKUBE_CONTAINER_GROUP=true make docker-compose
 

tools/docker-compose/README.md

@@ -465,14 +465,26 @@ Once the playbook is done running Splunk should now be setup in your development
 
 Prometheus is a metrics collecting tool, and we support prometheus formatted data at the `api/v2/metrics` endpoint.
 
-1. Change the `username` and `password` in `tools/prometheus/prometheus.yml`. You can also change the scrape interval.
+Before you run anything, you should perform this basic setup:
+
+1. Copy the prometheus configuration:
+
+```
+cp tools/prometheus/prometheus.yml.example tools/prometheus/prometheus.yml
+```
+
+Set the `username` and `password` in that file to your AWX user. You can also change the scrape interval.
+
 2. (optional) if you are in a clustered environment, you can change the target to `haproxy:8043` so that the incoming prometheus requests go through the load balancer. Leaving it set to `awx1` also works.
-3. run `make prometheus`
-4. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up.
-5. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button)
 
-Prometheus can display basic graphs of your data, but it is minimal. Often Prometheus is paired with an app like Grafana for better visualization features.
+You can use this as part of the docker-compose target:
 
-1. `make grafana` to run a local docker grafana instance.
-2. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password.
-4. Now you can create a dashboard and add panels for whichever metrics you like.
+```
+PROMETHEUS=true GRAFANA=true make docker-compose
+```
+
+3. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up.
+4. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button)
+5. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password.
+6. In the left navigation menu go to Dashboards->Browse, find the "awx-demo" and click. These should have graphs.
+6. Now you can modify these and add panels for whichever metrics you like.

tools/docker-compose/ansible/roles/sources/defaults/main.yml

@@ -29,3 +29,5 @@ ldap_private_key_file: '{{ ldap_cert_dir }}/{{ ldap_private_key_file_name }}'
 ldap_cert_subject: "/C=US/ST=NC/L=Durham/O=awx/CN="
 
 enable_splunk: false
+enable_grafana: false
+enable_prometheus: false

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -135,6 +135,37 @@ services:
     environment:
       SPLUNK_START_ARGS: --accept-license
       SPLUNK_PASSWORD: splunk_admin
+{% endif %}
+{% if enable_prometheus|bool %}
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: tools_prometheus_1
+    hostname: splunk
+    ports:
+      - "9090:9090"
+    volumes:
+      - "../../prometheus:/etc/prometheus"
+      - "prometheus_storage:/prometheus:rw"
+    links:
+      - awx_1:awx1
+{% endif %}
+{% if enable_grafana|bool %}
+  grafana:
+    image: grafana/grafana-enterprise:latest
+    container_name: tools_grafana_1
+    hostname: splunk
+    ports:
+      - "3001:3000"
+    volumes:
+      - "../../grafana:/etc/grafana/provisioning"
+      - "grafana_storage:/var/lib/grafana:rw"
+    environment:
+      SPLUNK_START_ARGS: --accept-license
+      SPLUNK_PASSWORD: splunk_admin
+    links:
+      - prometheus
+    depends_on:
+      - prometheus
 {% endif %}
   # A useful container that simply passes through log messages to the console
   # helpful for testing awx/tower logging
@@ -198,6 +229,14 @@ volumes:
     name: tools_ldap_1
     driver: local
 {% endif %}
+{% if enable_prometheus|bool %}
+  prometheus_storage:
+    name: tools_prometheus_storage
+{% endif %}
+{% if enable_grafana|bool %}
+  grafana_storage:
+    name: tools_grafana_storage
+{% endif %}
 {% if minikube_container_group|bool %}
 networks:
   default:

tools/prometheus/prometheus.yml.example

@@ -14,5 +14,5 @@ scrape_configs:
     params:
       format: ['txt']
     basic_auth:
-      username: awxuser  # change this
+      username: admin  # change this
       password: password  # change this