From 6f1f64118b6fcb69b9dabd6173b075d158465109 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 3 Mar 2021 15:38:27 -0500 Subject: [PATCH 1/4] wip --- Makefile | 12 ++-- tools/docker-compose/Dockerfile-haproxy | 2 - .../ansible/roles/sources/defaults/main.yml | 1 + .../sources/templates/docker-compose.yml.j2 | 65 +++++++++++++------ 4 files changed, 50 insertions(+), 30 deletions(-) delete mode 100644 tools/docker-compose/Dockerfile-haproxy diff --git a/Makefile b/Makefile index 858a6f67e4..971baae552 100644 --- a/Makefile +++ b/Makefile @@ -539,25 +539,23 @@ docker-compose-sources: ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \ -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \ -e awx_image_tag=$(COMPOSE_TAG) + -e cluster_node_count=$(CLUSER_NODE_COUNT) docker-compose: docker-auth awx/projects docker-compose-sources - docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up --no-recreate awx - -docker-compose-cluster: docker-auth awx/projects - docker-compose -f tools/docker-compose-cluster.yml up + docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up --no-recreate awx_1 docker-compose-credential-plugins: docker-auth awx/projects docker-compose-sources echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m" docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx docker-compose-test: docker-auth awx/projects docker-compose-sources - docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx /bin/bash + docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx_1 /bin/bash docker-compose-runtest: awx/projects docker-compose-sources - docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx /start_tests.sh + docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx_1 /start_tests.sh docker-compose-build-swagger: awx/projects docker-compose-sources - docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports --no-deps awx /start_tests.sh swagger + docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports --no-deps awx_1 /start_tests.sh swagger detect-schema-change: genschema curl https://s3.amazonaws.com/awx-public-ci-files/schema.json -o reference-schema.json diff --git a/tools/docker-compose/Dockerfile-haproxy b/tools/docker-compose/Dockerfile-haproxy deleted file mode 100644 index 9d38924939..0000000000 --- a/tools/docker-compose/Dockerfile-haproxy +++ /dev/null @@ -1,2 +0,0 @@ -FROM haproxy:1.6-alpine -COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg diff --git a/tools/docker-compose/ansible/roles/sources/defaults/main.yml b/tools/docker-compose/ansible/roles/sources/defaults/main.yml index f867d7ef2a..cff661deb8 100644 --- a/tools/docker-compose/ansible/roles/sources/defaults/main.yml +++ b/tools/docker-compose/ansible/roles/sources/defaults/main.yml @@ -5,3 +5,4 @@ awx_image: 'quay.io/ansible/awx_devel' pg_port: 5432 pg_username: 'awx' pg_database: 'awx' +cluster_node_count: "{{ lookup('env', 'CLUSTER_COUNT') | default(1, True) }}" diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 1d0799c7b9..0f043cf215 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -1,30 +1,34 @@ --- version: '2' services: +{% for i in range(cluster_node_count|int) %} + {% set container_postfix = loop.index %} + {% set awx_sdb_port_start = 7899 + (loop.index0*100) | int %} + {% set awx_sdb_port_end = 7999 + (loop.index0*100) | int %} # Primary AWX Development Container - awx: + awx_{{ container_postfix }}: user: "{{ ansible_user_uid }}" image: "{{ awx_image }}:{{ awx_image_tag }}" - container_name: tools_awx_1 - hostname: awx + container_name: tools_awx_{{ container_postfix }} + hostname: awx_{{ container_postfix }} command: launch_awx.sh environment: OS: "{{ os_info.stdout }}" SDB_HOST: 0.0.0.0 - SDB_PORT: 7899 + SDB_PORT: {{ awx_sdb_port_start }} AWX_GROUP_QUEUES: tower RECEPTORCTL_SOCKET: /var/run/receptor/receptor.sock ports: - - "2222:2222" - - "8888:8888" - - "8080:8080" - - "8013:8013" - - "8043:8043" - - "6899:6899" # default port range for sdb-listen - - "7899-7999:7899-7999" # default port range for sdb-listen +{% if cluster_node_count|int == 1 %} + - "8080:8080" # unused but mapped for debugging + - "8888:8888" # jupyter notebook + - "8013:8013" # http + - "8043:8043" # https +{% endif %} + - "{{ awx_sdb_port_start }}:{{ awx_sdb_port_end }}" # sdb-listen links: - postgres - - redis + - redis_{{ container_postfix }} working_dir: "/awx_devel" volumes: - "../../../:/awx_devel" @@ -34,12 +38,21 @@ services: - "../../docker-compose/_sources/local_settings.py:/etc/tower/conf.d/local_settings.py" - "../../docker-compose/_sources/SECRET_KEY:/etc/tower/SECRET_KEY" - "../../docker-compose/receptor.conf:/etc/receptor/receptor.conf" - - "redis_socket:/var/run/redis/:rw" - "receptor:/var/run/receptor/" - "/sys/fs/cgroup:/sys/fs/cgroup" - "~/.kube/config:/var/lib/awx/.kube/config" + - "redis_socket_{{ container_postfix }}:/var/run/redis/:rw" privileged: true tty: true + redis_{{ container_postfix }}: + image: redis:latest + container_name: tools_redis_{{ container_postfix }} + volumes: + - "../../redis/redis.conf:/usr/local/etc/redis/redis.conf" + - "redis_socket_{{ container_postfix }}:/var/run/redis/:rw" + entrypoint: ["redis-server"] + command: ["/usr/local/etc/redis/redis.conf"] +{% endfor %} # A useful container that simply passes through log messages to the console # helpful for testing awx/tower logging # logstash: @@ -56,16 +69,26 @@ services: POSTGRES_PASSWORD: {{ pg_password }} volumes: - "awx_db:/var/lib/postgresql/data" - redis: - image: redis:latest - container_name: tools_redis_1 +{% if cluster_node_count|int > 1 %} + haproxy: + image: haproxy volumes: - - "../../redis/redis.conf:/usr/local/etc/redis/redis.conf" - - "redis_socket:/var/run/redis/:rw" - entrypoint: ["redis-server"] - command: ["/usr/local/etc/redis/redis.conf"] + - "../ha.conf:/usr/local/etc/haproxy/haproxy.cfg" + depends_on: +{% for i in range(cluster_node_count|int) -%} + {% set container_postfix = loop.index %} + - "awx_{{ container_postfix }}" +{% endfor %} + ports: + - "8013:8013" + - "8043:8043" + - "1936:1936" +{% endif -%} {# haproxy #} volumes: awx_db: - redis_socket: receptor: +{% for i in range(cluster_node_count|int) -%} + {% set container_postfix = loop.index %} + redis_socket_{{ container_postfix }}: +{% endfor -%} From 69edef430c7ce5fa59c6dfe1195e39edf4a54efb Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 3 Mar 2021 19:41:50 -0500 Subject: [PATCH 2/4] Get clustered dev env working --- Makefile | 5 ++- .../ansible/roles/sources/tasks/main.yml | 1 + .../sources/templates/docker-compose.yml.j2 | 38 +++++++++++-------- .../roles/sources/templates/haproxy.cfg.j2} | 15 ++++---- 4 files changed, 34 insertions(+), 25 deletions(-) rename tools/docker-compose/{haproxy.cfg => ansible/roles/sources/templates/haproxy.cfg.j2} (67%) diff --git a/Makefile b/Makefile index 971baae552..a68ab74c46 100644 --- a/Makefile +++ b/Makefile @@ -534,15 +534,16 @@ awx/projects: @mkdir -p $@ COMPOSE_UP_OPTS ?= +CLUSER_NODE_COUNT ?= 1 docker-compose-sources: ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \ -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \ - -e awx_image_tag=$(COMPOSE_TAG) + -e awx_image_tag=$(COMPOSE_TAG) \ -e cluster_node_count=$(CLUSER_NODE_COUNT) docker-compose: docker-auth awx/projects docker-compose-sources - docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up --no-recreate awx_1 + docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up docker-compose-credential-plugins: docker-auth awx/projects docker-compose-sources echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m" diff --git a/tools/docker-compose/ansible/roles/sources/tasks/main.yml b/tools/docker-compose/ansible/roles/sources/tasks/main.yml index 33ace141f1..55937b4051 100644 --- a/tools/docker-compose/ansible/roles/sources/tasks/main.yml +++ b/tools/docker-compose/ansible/roles/sources/tasks/main.yml @@ -44,6 +44,7 @@ with_items: - "database.py" - "websocket_secret.py" + - "haproxy.cfg" - name: Delete old local_settings.py file: diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 0f043cf215..7d724c82f4 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -44,6 +44,28 @@ services: - "redis_socket_{{ container_postfix }}:/var/run/redis/:rw" privileged: true tty: true + ports: + - "{{ awx_sdb_port_start }}:{{ awx_sdb_port_end }}" # sdb-listen +{% if cluster_node_count|int == 1 %} + - "8080:8080" # unused but mapped for debugging + - "8888:8888" # jupyter notebook + - "8013:8013" # http + - "8043:8043" # https +{% else %} + haproxy: + image: haproxy + volumes: + - "./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg" + depends_on: +{% for i in range(cluster_node_count|int) -%} + {% set container_postfix = loop.index %} + - "awx_{{ container_postfix }}" +{% endfor %} + ports: + - "8013:8013" + - "8043:8043" + - "1936:1936" +{% endif %} redis_{{ container_postfix }}: image: redis:latest container_name: tools_redis_{{ container_postfix }} @@ -69,22 +91,6 @@ services: POSTGRES_PASSWORD: {{ pg_password }} volumes: - "awx_db:/var/lib/postgresql/data" -{% if cluster_node_count|int > 1 %} - haproxy: - image: haproxy - volumes: - - "../ha.conf:/usr/local/etc/haproxy/haproxy.cfg" - depends_on: -{% for i in range(cluster_node_count|int) -%} - {% set container_postfix = loop.index %} - - "awx_{{ container_postfix }}" -{% endfor %} - ports: - - "8013:8013" - - "8043:8043" - - "1936:1936" -{% endif -%} {# haproxy #} - volumes: awx_db: receptor: diff --git a/tools/docker-compose/haproxy.cfg b/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2 similarity index 67% rename from tools/docker-compose/haproxy.cfg rename to tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2 index d37cbf691a..33837ded1f 100644 --- a/tools/docker-compose/haproxy.cfg +++ b/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2 @@ -1,5 +1,4 @@ global - debug stats socket /tmp/admin.sock stats timeout 30s @@ -30,16 +29,18 @@ backend nodes http-request set-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto https if { ssl_fc } option httpchk HEAD / HTTP/1.1\r\nHost:localhost - server awx-1 awx-1:8013 check - server awx-2 awx-2:8013 check - server awx-3 awx-3:8013 check +{% for i in range(cluster_node_count|int) %} + {% set container_postfix = loop.index %} + server tools_awx_{{ container_postfix }} tools_awx_{{ container_postfix }}:8013 check +{% endfor %} backend nodes_ssl mode tcp balance roundrobin - server awx-1 awx-1:8043 - server awx-2 awx-2:8043 - server awx-3 awx-3:8043 +{% for i in range(cluster_node_count|int) %} + {% set container_postfix = loop.index %} + server tools_awx_{{ container_postfix }} tools_awx_{{ container_postfix }}:8043 check +{% endfor %} listen stats bind *:1936 From 7b1edda368f82eeaca13bf75f22eada0b65b0a36 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 4 Mar 2021 15:04:36 -0500 Subject: [PATCH 3/4] support receptor in multi cluster nodes --- Makefile | 2 +- .../ansible/roles/sources/templates/docker-compose.yml.j2 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a68ab74c46..a06c32c2de 100644 --- a/Makefile +++ b/Makefile @@ -540,7 +540,7 @@ docker-compose-sources: ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \ -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \ -e awx_image_tag=$(COMPOSE_TAG) \ - -e cluster_node_count=$(CLUSER_NODE_COUNT) + -e cluster_node_count=$(CLUSTER_NODE_COUNT) docker-compose: docker-auth awx/projects docker-compose-sources docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 7d724c82f4..ccc42d2b0e 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -38,10 +38,10 @@ services: - "../../docker-compose/_sources/local_settings.py:/etc/tower/conf.d/local_settings.py" - "../../docker-compose/_sources/SECRET_KEY:/etc/tower/SECRET_KEY" - "../../docker-compose/receptor.conf:/etc/receptor/receptor.conf" - - "receptor:/var/run/receptor/" - "/sys/fs/cgroup:/sys/fs/cgroup" - "~/.kube/config:/var/lib/awx/.kube/config" - "redis_socket_{{ container_postfix }}:/var/run/redis/:rw" + - "receptor_{{ container_postfix }}:/var/run/receptor/" privileged: true tty: true ports: @@ -93,8 +93,8 @@ services: - "awx_db:/var/lib/postgresql/data" volumes: awx_db: - receptor: {% for i in range(cluster_node_count|int) -%} {% set container_postfix = loop.index %} + receptor_{{ container_postfix }}: redis_socket_{{ container_postfix }}: {% endfor -%} From 16a6fb5adc96d991d7e069142b0374b9342abde9 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 4 Mar 2021 15:23:04 -0500 Subject: [PATCH 4/4] add docs for cluster dev mode --- tools/docker-compose/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/docker-compose/README.md b/tools/docker-compose/README.md index 88a3ea1883..d381614a75 100644 --- a/tools/docker-compose/README.md +++ b/tools/docker-compose/README.md @@ -170,3 +170,15 @@ In order to launch all developer services: `launch_awx.sh` also calls `bootstrap_development.sh` so if all you are doing is launching the supervisor to start all services, you don't need to call `bootstrap_development.sh` first. + +### Start a cluster + +Certain features or bugs are only applicable when running a cluster of AWX nodes. To bring up a 3 node cluster development environment simply run the below command. + +```bash +(host)$ CLUSTER_NODE_COUNT=3 make docker-compose +``` + +`CLUSTER_NODE_COUNT` is configurable and defaults to 1, effectively a non-clustered AWX. + +Note that you may see multiple messages of the form `2021-03-04 20:11:47,666 WARNING [-] awx.main.wsbroadcast Connection from awx_2 to awx_5 failed: 'Cannot connect to host awx_5:8013 ssl:False [Name or service not known]'.`. This can happen when you bring up a cluster of many nodes, say 10, then you bring up a cluster of less nodes, say 3. In this example, there will be 7 `Instance` records in the database that represent AWX instances. The AWX development environment mimics the VM deployment (vs. kubernetes) and expects the missing nodes to be brought back to healthy by the admin. The warning message you are seeing is all of the AWX nodes trying to connect the websocket backplane. You can manually delete the `Instance` records from the database i.e. `Instance.objects.get(hostname='awx_9').delete()` to stop the warnings.