Merge pull request #9471 from chrismeyersfsu/fix-docker_cluster

fix up awx docker cluster

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

Makefile

@@ -534,30 +534,29 @@ awx/projects:
 	@mkdir -p $@
 
 COMPOSE_UP_OPTS ?=
+CLUSER_NODE_COUNT ?= 1
 
 docker-compose-sources:
 	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \
 	    -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \
-	    -e awx_image_tag=$(COMPOSE_TAG)
+	    -e awx_image_tag=$(COMPOSE_TAG) \
+	    -e cluster_node_count=$(CLUSTER_NODE_COUNT)
 
 docker-compose: docker-auth awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up --no-recreate awx
+	docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up
-
-docker-compose-cluster: docker-auth awx/projects
-	docker-compose -f tools/docker-compose-cluster.yml up
 
 docker-compose-credential-plugins: docker-auth awx/projects docker-compose-sources
 	echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m"
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx
 
 docker-compose-test: docker-auth awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx /bin/bash
+	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx_1 /bin/bash
 
 docker-compose-runtest: awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx /start_tests.sh
+	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx_1 /start_tests.sh
 
 docker-compose-build-swagger: awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports --no-deps awx /start_tests.sh swagger
+	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports --no-deps awx_1 /start_tests.sh swagger
 
 detect-schema-change: genschema
 	curl https://s3.amazonaws.com/awx-public-ci-files/schema.json -o reference-schema.json

tools/docker-compose/Dockerfile-haproxy

@@ -1,2 +0,0 @@
-FROM haproxy:1.6-alpine
-COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg

tools/docker-compose/README.md

@@ -170,3 +170,15 @@ In order to launch all developer services:
 
 `launch_awx.sh` also calls `bootstrap_development.sh` so if all you are doing is launching the supervisor to start all services, you don't
 need to call `bootstrap_development.sh` first.
+
+### Start a cluster
+
+Certain features or bugs are only applicable when running a cluster of AWX nodes. To bring up a 3 node cluster development environment simply run the below command.
+
+```bash
+(host)$ CLUSTER_NODE_COUNT=3 make docker-compose
+```
+
+`CLUSTER_NODE_COUNT` is configurable and defaults to 1, effectively a non-clustered AWX.
+
+Note that you may see multiple messages of the form `2021-03-04 20:11:47,666 WARNING  [-] awx.main.wsbroadcast Connection from awx_2 to awx_5 failed: 'Cannot connect to host awx_5:8013 ssl:False [Name or service not known]'.`. This can happen when you bring up a cluster of many nodes, say 10, then you bring up a cluster of less nodes, say 3. In this example, there will be 7 `Instance` records in the database that represent AWX instances. The AWX development environment mimics the VM deployment (vs. kubernetes) and expects the missing nodes to be brought back to healthy by the admin. The warning message you are seeing is all of the AWX nodes trying to connect the websocket backplane. You can manually delete the `Instance` records from the database i.e. `Instance.objects.get(hostname='awx_9').delete()` to stop the warnings.

tools/docker-compose/ansible/roles/sources/defaults/main.yml

@@ -5,3 +5,4 @@ awx_image: 'quay.io/ansible/awx_devel'
 pg_port: 5432
 pg_username: 'awx'
 pg_database: 'awx'
+cluster_node_count: "{{ lookup('env', 'CLUSTER_COUNT') | default(1, True) }}"

tools/docker-compose/ansible/roles/sources/tasks/main.yml

@@ -44,6 +44,7 @@
   with_items:
     - "database.py"
     - "websocket_secret.py"
+    - "haproxy.cfg"
 
 - name: Delete old local_settings.py
   file:

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -1,30 +1,34 @@
 ---
 version: '2'
 services:
+{% for i in range(cluster_node_count|int) %}
+  {% set container_postfix = loop.index %}
+  {% set awx_sdb_port_start = 7899 + (loop.index0*100) | int %}
+  {% set awx_sdb_port_end = 7999 + (loop.index0*100) | int %}
   # Primary AWX Development Container
-  awx:
+  awx_{{ container_postfix }}:
     user: "{{ ansible_user_uid }}"
     image: "{{ awx_image }}:{{ awx_image_tag }}"
-    container_name: tools_awx_1
+    container_name: tools_awx_{{ container_postfix }}
-    hostname: awx
+    hostname: awx_{{ container_postfix }}
     command: launch_awx.sh
     environment:
       OS: "{{ os_info.stdout }}"
       SDB_HOST: 0.0.0.0
-      SDB_PORT: 7899
+      SDB_PORT: {{ awx_sdb_port_start }}
       AWX_GROUP_QUEUES: tower
       RECEPTORCTL_SOCKET: /var/run/receptor/receptor.sock
     ports:
-      - "2222:2222"
+{% if cluster_node_count|int == 1 %}
-      - "8888:8888"
+      - "8080:8080"  # unused but mapped for debugging
-      - "8080:8080"
+      - "8888:8888"  # jupyter notebook
-      - "8013:8013"
+      - "8013:8013"  # http
-      - "8043:8043"
+      - "8043:8043"  # https
-      - "6899:6899"  # default port range for sdb-listen
+{% endif %}
-      - "7899-7999:7899-7999"  # default port range for sdb-listen
+      - "{{ awx_sdb_port_start }}:{{ awx_sdb_port_end }}"  # sdb-listen
     links:
       - postgres
-      - redis
+      - redis_{{ container_postfix }}
     working_dir: "/awx_devel"
     volumes:
       - "../../../:/awx_devel"
@@ -34,12 +38,43 @@ services:
       - "../../docker-compose/_sources/local_settings.py:/etc/tower/conf.d/local_settings.py"
       - "../../docker-compose/_sources/SECRET_KEY:/etc/tower/SECRET_KEY"
       - "../../docker-compose/receptor.conf:/etc/receptor/receptor.conf"
-      - "redis_socket:/var/run/redis/:rw"
-      - "receptor:/var/run/receptor/"
       - "/sys/fs/cgroup:/sys/fs/cgroup"
       - "~/.kube/config:/var/lib/awx/.kube/config"
+      - "redis_socket_{{ container_postfix }}:/var/run/redis/:rw"
+      - "receptor_{{ container_postfix }}:/var/run/receptor/"
     privileged: true
     tty: true
+    ports:
+      - "{{ awx_sdb_port_start }}:{{ awx_sdb_port_end }}"  # sdb-listen
+{% if cluster_node_count|int == 1 %}
+      - "8080:8080"  # unused but mapped for debugging
+      - "8888:8888"  # jupyter notebook
+      - "8013:8013"  # http
+      - "8043:8043"  # https
+{% else %}
+  haproxy:
+    image: haproxy
+    volumes:
+      - "./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg"
+    depends_on:
+{% for i in range(cluster_node_count|int) -%}
+  {% set container_postfix = loop.index %}
+      - "awx_{{ container_postfix }}"
+{% endfor %}
+    ports:
+      - "8013:8013"
+      - "8043:8043"
+      - "1936:1936"
+{% endif %}
+  redis_{{ container_postfix }}:
+    image: redis:latest
+    container_name: tools_redis_{{ container_postfix }}
+    volumes:
+      - "../../redis/redis.conf:/usr/local/etc/redis/redis.conf"
+      - "redis_socket_{{ container_postfix }}:/var/run/redis/:rw"
+    entrypoint: ["redis-server"]
+    command: ["/usr/local/etc/redis/redis.conf"]
+{% endfor %}
   # A useful container that simply passes through log messages to the console
   # helpful for testing awx/tower logging
   # logstash:
@@ -56,16 +91,10 @@ services:
       POSTGRES_PASSWORD: {{ pg_password }}
     volumes:
       - "awx_db:/var/lib/postgresql/data"
-  redis:
-    image: redis:latest
-    container_name: tools_redis_1
-    volumes:
-      - "../../redis/redis.conf:/usr/local/etc/redis/redis.conf"
-      - "redis_socket:/var/run/redis/:rw"
-    entrypoint: ["redis-server"]
-    command: ["/usr/local/etc/redis/redis.conf"]
-
 volumes:
   awx_db:
-  redis_socket:
+{% for i in range(cluster_node_count|int) -%}
-  receptor:
+  {% set container_postfix = loop.index %}
+  receptor_{{ container_postfix }}:
+  redis_socket_{{ container_postfix }}:
+{% endfor -%}

tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2

@@ -1,5 +1,4 @@
 global
-    debug
     stats socket /tmp/admin.sock
     stats timeout 30s
 
@@ -30,16 +29,18 @@ backend nodes
     http-request set-header X-Forwarded-Port %[dst_port]
     http-request add-header X-Forwarded-Proto https if { ssl_fc }
     option httpchk HEAD / HTTP/1.1\r\nHost:localhost
-    server awx-1 awx-1:8013 check
+{% for i in range(cluster_node_count|int) %}
-    server awx-2 awx-2:8013 check
+    {% set container_postfix = loop.index %}
-    server awx-3 awx-3:8013 check
+    server tools_awx_{{ container_postfix }} tools_awx_{{ container_postfix }}:8013 check
+{% endfor %}
 
 backend nodes_ssl
     mode tcp
     balance roundrobin
-    server awx-1 awx-1:8043
+{% for i in range(cluster_node_count|int) %}
-    server awx-2 awx-2:8043
+    {% set container_postfix = loop.index %}
-    server awx-3 awx-3:8043
+    server tools_awx_{{ container_postfix }} tools_awx_{{ container_postfix }}:8043 check
+{% endfor %}
 
 listen stats
     bind *:1936