From fb292d97061aaf826995d5ae9b9188a33d2f6d8b Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 16 Jun 2022 10:25:02 -0400 Subject: [PATCH 1/7] Move visualization containers into docker-compose --- Makefile | 4 +- tools/docker-compose/README.md | 16 ++++++-- .../ansible/roles/sources/defaults/main.yml | 2 + .../sources/templates/docker-compose.yml.j2 | 37 +++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 8662626d31..411d9dbf2e 100644 --- a/Makefile +++ b/Makefile @@ -470,7 +470,9 @@ docker-compose-sources: .git/hooks/pre-commit -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP) \ -e enable_keycloak=$(KEYCLOAK) \ -e enable_ldap=$(LDAP) \ - -e enable_splunk=$(SPLUNK) + -e enable_splunk=$(SPLUNK) \ + -e enable_prometheus=$(PROMETHEUS) \ + -e enable_grafana=$(GRAFANA) docker-compose: awx/projects docker-compose-sources diff --git a/tools/docker-compose/README.md b/tools/docker-compose/README.md index 45763c76f1..28457c0eed 100644 --- a/tools/docker-compose/README.md +++ b/tools/docker-compose/README.md @@ -404,7 +404,7 @@ Anytime you want to run an OpenLDAP instance alongside AWX we can start docker-c LDAP=true make docker-compose ``` -Once the containers come up two new ports (389, 636) should be exposed and the LDAP server should be running on those ports. The first port (389) is non-SSL and the second port (636) is SSL enabled. +Once the containers come up two new ports (389, 636) should be exposed and the LDAP server should be running on those ports. The first port (389) is non-SSL and the second port (636) is SSL enabled. Now we are ready to configure and plumb OpenLDAP with AWX. To do this we have provided a playbook which will: * Backup and configure the LDAP adapter in AWX. NOTE: this will back up your existing settings but the password fields can not be backed up through the API, you need a DB backup to recover this. @@ -427,7 +427,7 @@ Once the playbook is done running LDAP should now be setup in your development e 3. awx_ldap_auditor:audit123 4. awx_ldap_org_admin:orgadmin123 -The first account is a normal user. The second account will be a super user in AWX. The third account will be a system auditor in AWX. The fourth account is an org admin. All users belong to an org called "LDAP Organization". To log in with one of these users go to the AWX login screen enter the username/password. +The first account is a normal user. The second account will be a super user in AWX. The third account will be a system auditor in AWX. The fourth account is an org admin. All users belong to an org called "LDAP Organization". To log in with one of these users go to the AWX login screen enter the username/password. ### Splunk Integration @@ -449,7 +449,7 @@ Once the containers are up we are ready to configure and plumb Splunk with AWX. * Backup and configure the External Logging adapter in AWX. NOTE: this will back up your existing settings but the password fields can not be backed up through the API, you need a DB backup to recover this. * Create a TCP port in Splunk for log forwarding -For routing traffic between AWX and Splunk we will use the internal docker compose network. The `Logging Aggregator` will be configured using the internal network machine name of `splunk`. +For routing traffic between AWX and Splunk we will use the internal docker compose network. The `Logging Aggregator` will be configured using the internal network machine name of `splunk`. Once you have have the collections installed (from above) you can run the playbook like: ```bash @@ -464,6 +464,16 @@ Once the playbook is done running Splunk should now be setup in your development ### Prometheus and Grafana integration Prometheus is a metrics collecting tool, and we support prometheus formatted data at the `api/v2/metrics` endpoint. +You can use this as part of the docker-compose target: + +``` +PROMETHEUS=true GRAFANA=true make docker-compose +``` + +TODO, internal, delete +PROMETHEUS=true GRAFANA=true MAIN_NODE_TYPE=hybrid EXECUTION_NODE_COUNT=0 COMPOSE_TAG=devel make docker-compose + +Alternatively, you can run as separate commands (deprecated, may be deleted in future). 1. Change the `username` and `password` in `tools/prometheus/prometheus.yml`. You can also change the scrape interval. 2. (optional) if you are in a clustered environment, you can change the target to `haproxy:8043` so that the incoming prometheus requests go through the load balancer. Leaving it set to `awx1` also works. diff --git a/tools/docker-compose/ansible/roles/sources/defaults/main.yml b/tools/docker-compose/ansible/roles/sources/defaults/main.yml index 364b7da3da..e1a67437d9 100644 --- a/tools/docker-compose/ansible/roles/sources/defaults/main.yml +++ b/tools/docker-compose/ansible/roles/sources/defaults/main.yml @@ -29,3 +29,5 @@ ldap_private_key_file: '{{ ldap_cert_dir }}/{{ ldap_private_key_file_name }}' ldap_cert_subject: "/C=US/ST=NC/L=Durham/O=awx/CN=" enable_splunk: false +enable_grafana: false +enable_prometheus: false diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 99b439fdb2..e7a2d39ba9 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -135,6 +135,35 @@ services: environment: SPLUNK_START_ARGS: --accept-license SPLUNK_PASSWORD: splunk_admin +{% endif %} +{% if enable_prometheus|bool %} + prometheus: + image: prom/prometheus:latest + container_name: tools_prometheus_1 + hostname: splunk + ports: + - "9090:9090" + volumes: + - "../../prometheus:/etc/prometheus" + - "prometheus_storage:/prometheus:rw" + links: + - awx_1:awx1 +{% endif %} +{% if enable_grafana|bool %} + grafana: + image: grafana/grafana-enterprise:latest + container_name: tools_grafana_1 + hostname: splunk + ports: + - "3001:3000" + volumes: + - "../../grafana:/etc/grafana/provisioning" + - "grafana_storage:/var/lib/grafana:rw" + environment: + SPLUNK_START_ARGS: --accept-license + SPLUNK_PASSWORD: splunk_admin + links: + - prometheus {% endif %} # A useful container that simply passes through log messages to the console # helpful for testing awx/tower logging @@ -198,6 +227,14 @@ volumes: name: tools_ldap_1 driver: local {% endif %} +{% if enable_prometheus|bool %} + prometheus_storage: + name: tools_prometheus_storage +{% endif %} +{% if enable_grafana|bool %} + grafana_storage: + name: tools_grafana_storage +{% endif %} {% if minikube_container_group|bool %} networks: default: From c441db2aab788c277927c7df3b26030cfe37f27d Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 16 Jun 2022 12:07:26 -0400 Subject: [PATCH 2/7] docs workding edits and depends_on --- tools/docker-compose/README.md | 18 ++++++++++-------- .../sources/templates/docker-compose.yml.j2 | 2 ++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tools/docker-compose/README.md b/tools/docker-compose/README.md index 28457c0eed..78ba7ada53 100644 --- a/tools/docker-compose/README.md +++ b/tools/docker-compose/README.md @@ -464,25 +464,27 @@ Once the playbook is done running Splunk should now be setup in your development ### Prometheus and Grafana integration Prometheus is a metrics collecting tool, and we support prometheus formatted data at the `api/v2/metrics` endpoint. + +Before you run anything, you should perform this basic setup: + +1. Change the `username` and `password` in `tools/prometheus/prometheus.yml`. You can also change the scrape interval. +2. (optional) if you are in a clustered environment, you can change the target to `haproxy:8043` so that the incoming prometheus requests go through the load balancer. Leaving it set to `awx1` also works. + You can use this as part of the docker-compose target: ``` PROMETHEUS=true GRAFANA=true make docker-compose ``` - -TODO, internal, delete -PROMETHEUS=true GRAFANA=true MAIN_NODE_TYPE=hybrid EXECUTION_NODE_COUNT=0 COMPOSE_TAG=devel make docker-compose +Once you've done that, you should be able to navigate to http://localhost:9090/targets and http://localhost:3001. Alternatively, you can run as separate commands (deprecated, may be deleted in future). -1. Change the `username` and `password` in `tools/prometheus/prometheus.yml`. You can also change the scrape interval. -2. (optional) if you are in a clustered environment, you can change the target to `haproxy:8043` so that the incoming prometheus requests go through the load balancer. Leaving it set to `awx1` also works. 3. run `make prometheus` 4. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up. 5. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button) Prometheus can display basic graphs of your data, but it is minimal. Often Prometheus is paired with an app like Grafana for better visualization features. -1. `make grafana` to run a local docker grafana instance. -2. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password. -4. Now you can create a dashboard and add panels for whichever metrics you like. +6. `make grafana` to run a local docker grafana instance. +7. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password. +8. Now you can create a dashboard and add panels for whichever metrics you like. diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index e7a2d39ba9..eadfe0b976 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -164,6 +164,8 @@ services: SPLUNK_PASSWORD: splunk_admin links: - prometheus + depends_on: + - prometheus {% endif %} # A useful container that simply passes through log messages to the console # helpful for testing awx/tower logging From f4ef7d6927b2280c3216e44553cdeb3e45da4edd Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 16 Jun 2022 14:03:22 -0400 Subject: [PATCH 3/7] Add volumes to the clean command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 411d9dbf2e..c590642599 100644 --- a/Makefile +++ b/Makefile @@ -520,7 +520,7 @@ docker-clean: fi docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean - docker volume rm tools_awx_db + docker volume rm -f tools_awx_db tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q) docker-refresh: docker-clean docker-compose From 1dd535a859ed9833a54b58d99358e6461b683a8e Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 16 Jun 2022 15:31:45 -0400 Subject: [PATCH 4/7] Remove old way of doing grafana graphs --- Makefile | 8 -------- tools/docker-compose/README.md | 17 +++++------------ 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index c590642599..98344c8df8 100644 --- a/Makefile +++ b/Makefile @@ -531,14 +531,6 @@ docker-compose-elk: awx/projects docker-compose-sources docker-compose-cluster-elk: awx/projects docker-compose-sources docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate -prometheus: - docker volume create prometheus - docker run -d --rm --net=_sources_default --link=awx_1:awx1 --volume prometheus-storage:/prometheus --volume `pwd`/tools/prometheus:/etc/prometheus --name prometheus -p 9090:9090 prom/prometheus - -grafana: - docker volume create grafana - docker run -d --rm --net=_sources_default --volume grafana-storage:/var/lib/grafana --volume `pwd`/tools/grafana:/etc/grafana/provisioning --name grafana -p 3001:3000 grafana/grafana-enterprise - docker-compose-container-group: MINIKUBE_CONTAINER_GROUP=true make docker-compose diff --git a/tools/docker-compose/README.md b/tools/docker-compose/README.md index 78ba7ada53..1238bb5310 100644 --- a/tools/docker-compose/README.md +++ b/tools/docker-compose/README.md @@ -475,16 +475,9 @@ You can use this as part of the docker-compose target: ``` PROMETHEUS=true GRAFANA=true make docker-compose ``` -Once you've done that, you should be able to navigate to http://localhost:9090/targets and http://localhost:3001. -Alternatively, you can run as separate commands (deprecated, may be deleted in future). - -3. run `make prometheus` -4. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up. -5. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button) - -Prometheus can display basic graphs of your data, but it is minimal. Often Prometheus is paired with an app like Grafana for better visualization features. - -6. `make grafana` to run a local docker grafana instance. -7. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password. -8. Now you can create a dashboard and add panels for whichever metrics you like. +3. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up. +4. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button) +5. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password. +6. In the left navigation menu go to Dashboards->Browse, find the "awx-demo" and click. These should have graphs. +6. Now you can modify these and add panels for whichever metrics you like. From 173318764b1b32b2807a8e0d9dfe1f7b63497b54 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 16 Jun 2022 15:37:18 -0400 Subject: [PATCH 5/7] Remove existing yml file for prometheus --- tools/prometheus/prometheus.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 tools/prometheus/prometheus.yml diff --git a/tools/prometheus/prometheus.yml b/tools/prometheus/prometheus.yml deleted file mode 100644 index 52f40d2bb6..0000000000 --- a/tools/prometheus/prometheus.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -global: - scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. - -scrape_configs: - - job_name: 'awx' - static_configs: - - targets: ['awx1:8043'] # or haproxy:8043 in cluster env - tls_config: - insecure_skip_verify: true - metrics_path: /api/v2/metrics - scrape_interval: 5s - scheme: https - params: - format: ['txt'] - basic_auth: - username: awxuser # change this - password: password # change this From 372aa362076dd435870e2ace57bbd6e626171929 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 16 Jun 2022 15:41:06 -0400 Subject: [PATCH 6/7] Make the prometheus config file ignored by git --- .gitignore | 4 +++- tools/docker-compose/README.md | 9 ++++++++- tools/prometheus/prometheus.yml.example | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 tools/prometheus/prometheus.yml.example diff --git a/.gitignore b/.gitignore index a5d367158c..4c735efb47 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,6 @@ awx/ui/build awx/ui/.env.local awx/ui/instrumented rsyslog.pid -tools/prometheus tools/docker-compose/ansible/awx_dump.sql tools/docker-compose/Dockerfile tools/docker-compose/_build @@ -154,6 +153,9 @@ use_dev_supervisor.txt /sanity/ /awx_collection_build/ +# Setup for metrics gathering +tools/prometheus/prometheus.yml + .idea/* *.unison.tmp *.# diff --git a/tools/docker-compose/README.md b/tools/docker-compose/README.md index 1238bb5310..778c789cb4 100644 --- a/tools/docker-compose/README.md +++ b/tools/docker-compose/README.md @@ -467,7 +467,14 @@ Prometheus is a metrics collecting tool, and we support prometheus formatted dat Before you run anything, you should perform this basic setup: -1. Change the `username` and `password` in `tools/prometheus/prometheus.yml`. You can also change the scrape interval. +1. Copy the prometheus configuration: + +``` +cp tools/prometheus/prometheus.yml.example tools/prometheus/prometheus.yml +``` + +Set the `username` and `password` in that file to your AWX user. You can also change the scrape interval. + 2. (optional) if you are in a clustered environment, you can change the target to `haproxy:8043` so that the incoming prometheus requests go through the load balancer. Leaving it set to `awx1` also works. You can use this as part of the docker-compose target: diff --git a/tools/prometheus/prometheus.yml.example b/tools/prometheus/prometheus.yml.example new file mode 100644 index 0000000000..810bb56db0 --- /dev/null +++ b/tools/prometheus/prometheus.yml.example @@ -0,0 +1,18 @@ +--- +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + +scrape_configs: + - job_name: 'awx' + static_configs: + - targets: ['awx1:8043'] # or haproxy:8043 in cluster env + tls_config: + insecure_skip_verify: true + metrics_path: /api/v2/metrics + scrape_interval: 5s + scheme: https + params: + format: ['txt'] + basic_auth: + username: admin # change this + password: password # change this From db105c21e43c17db2105545749844b983298be0e Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 16 Jun 2022 15:46:42 -0400 Subject: [PATCH 7/7] Set default false values --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 98344c8df8..9295d697a2 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,10 @@ KEYCLOAK ?= false LDAP ?= false # If set to true docker-compose will also start a splunk instance SPLUNK ?= false +# If set to true docker-compose will also start a prometheus instance +PROMETHEUS ?= false +# If set to true docker-compose will also start a grafana instance +GRAFANA ?= false VENV_BASE ?= /var/lib/awx/venv