From 460c7c337921b855609b3eea6b7bc1157472ea61 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 22 Jul 2021 19:30:00 -0400 Subject: [PATCH] Allow for dynamically scaling automation mesh in dev env --- Makefile | 9 ++- tools/docker-compose-cluster/.gitignore | 0 .../awx-2-receptor.conf | 25 ------- .../awx-3-receptor.conf | 25 ------- .../awx-hop-receptor.conf | 26 -------- tools/docker-compose-cluster/certs/.gitignore | 2 - tools/docker-compose/README.md | 4 +- .../ansible/roles/sources/defaults/main.yml | 2 +- .../ansible/roles/sources/tasks/main.yml | 20 ++++-- .../sources/templates/docker-compose.yml.j2 | 53 +++++++++------ .../sources/templates/docker-receptor.yml.j2 | 66 ------------------- .../roles/sources/templates/haproxy.cfg.j2 | 4 +- .../sources/templates/receptor-hop.conf.j2 | 12 ++++ .../templates/receptor-worker.conf.j2} | 9 +-- 14 files changed, 71 insertions(+), 186 deletions(-) delete mode 100644 tools/docker-compose-cluster/.gitignore delete mode 100644 tools/docker-compose-cluster/awx-2-receptor.conf delete mode 100644 tools/docker-compose-cluster/awx-3-receptor.conf delete mode 100644 tools/docker-compose-cluster/awx-hop-receptor.conf delete mode 100644 tools/docker-compose-cluster/certs/.gitignore delete mode 100644 tools/docker-compose/ansible/roles/sources/templates/docker-receptor.yml.j2 create mode 100644 tools/docker-compose/ansible/roles/sources/templates/receptor-hop.conf.j2 rename tools/{docker-compose-cluster/awx-1-receptor.conf => docker-compose/ansible/roles/sources/templates/receptor-worker.conf.j2} (53%) diff --git a/Makefile b/Makefile index f6a0675323..f8269bdd1b 100644 --- a/Makefile +++ b/Makefile @@ -479,7 +479,8 @@ awx/projects: COMPOSE_UP_OPTS ?= COMPOSE_OPTS ?= -CLUSTER_NODE_COUNT ?= 1 +CONTROL_PLANE_NODE_COUNT ?= 1 +EXECUTION_NODE_COUNT ?= 2 MINIKUBE_CONTAINER_GROUP ?= false docker-compose-sources: .git/hooks/pre-commit @@ -490,7 +491,8 @@ docker-compose-sources: .git/hooks/pre-commit ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \ -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \ -e awx_image_tag=$(COMPOSE_TAG) \ - -e cluster_node_count=$(CLUSTER_NODE_COUNT) \ + -e control_plane_node_count=$(CONTROL_PLANE_NODE_COUNT) \ + -e execution_node_count=$(EXECUTION_NODE_COUNT) \ -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP) @@ -544,9 +546,6 @@ docker-refresh: docker-clean docker-compose docker-compose-elk: docker-auth awx/projects docker-compose-sources docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate -docker-compose-cluster: docker-auth awx/projects docker-compose-sources - docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-compose/_sources/docker-receptor.yml up - docker-compose-cluster-elk: docker-auth awx/projects docker-compose-sources docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate diff --git a/tools/docker-compose-cluster/.gitignore b/tools/docker-compose-cluster/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/docker-compose-cluster/awx-2-receptor.conf b/tools/docker-compose-cluster/awx-2-receptor.conf deleted file mode 100644 index af85fe0ea6..0000000000 --- a/tools/docker-compose-cluster/awx-2-receptor.conf +++ /dev/null @@ -1,25 +0,0 @@ ---- -- node: - id: receptor-2 - -- log-level: info - -#- tls-client: -# name: mutual-tls-client -# cert: /etc/receptor/certs/receptor-2.crt -# key: /etc/receptor/certs/receptor-2.key -# rootcas: /etc/receptor/certs/ca.crt - -- tcp-peer: - address: tools_receptor_hop:5555 - redial: true - #tls: mutual-tls-client - -- work-command: - worktype: ansible-runner - command: ansible-runner - params: worker - allowruntimeparams: true - -- control-service: - service: control diff --git a/tools/docker-compose-cluster/awx-3-receptor.conf b/tools/docker-compose-cluster/awx-3-receptor.conf deleted file mode 100644 index 095ff8fee9..0000000000 --- a/tools/docker-compose-cluster/awx-3-receptor.conf +++ /dev/null @@ -1,25 +0,0 @@ ---- -- node: - id: receptor-3 - -- log-level: info - -#- tls-client: -# name: mutual-tls-client -# cert: /etc/receptor/certs/receptor-3.crt -# key: /etc/receptor/certs/receptor-3.key -# rootcas: /etc/receptor/certs/ca.crt - -- tcp-peer: - address: tools_receptor_hop:5555 - redial: true - #tls: mutual-tls-client - -- work-command: - worktype: ansible-runner - command: ansible-runner - params: worker - allowruntimeparams: true - -- control-service: - service: control diff --git a/tools/docker-compose-cluster/awx-hop-receptor.conf b/tools/docker-compose-cluster/awx-hop-receptor.conf deleted file mode 100644 index 07a1140e66..0000000000 --- a/tools/docker-compose-cluster/awx-hop-receptor.conf +++ /dev/null @@ -1,26 +0,0 @@ ---- -- node: - id: receptor-hop - -- log-level: info - -#- tls-server: -# name: mutual-tls-server -# cert: /etc/receptor/certs/receptor-hop.crt -# key: /etc/receptor/certs/receptor-hop.key -# clientcas: /etc/receptor/certs/ca.crt - -- tls-client: - name: mutual-tls-client - cert: /etc/receptor/certs/receptor-hop.crt - key: /etc/receptor/certs/receptor-hop.key - rootcas: /etc/receptor/certs/ca.crt - -- tcp-peer: - address: awx_1:4444 - redial: true - #tls: mutual-tls-client - -- tcp-listener: - port: 5555 - #tls: mutual-tls-server diff --git a/tools/docker-compose-cluster/certs/.gitignore b/tools/docker-compose-cluster/certs/.gitignore deleted file mode 100644 index d6b7ef32c8..0000000000 --- a/tools/docker-compose-cluster/certs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/tools/docker-compose/README.md b/tools/docker-compose/README.md index 7396dd9326..13e7a5be21 100644 --- a/tools/docker-compose/README.md +++ b/tools/docker-compose/README.md @@ -211,10 +211,10 @@ need to call `bootstrap_development.sh` first. Certain features or bugs are only applicable when running a cluster of AWX nodes. To bring up a 3 node cluster development environment simply run the below command. ```bash -(host)$ CLUSTER_NODE_COUNT=3 make docker-compose +(host)$ CONTROL_PLANE_NODE_COUNT=3 make docker-compose ``` -`CLUSTER_NODE_COUNT` is configurable and defaults to 1, effectively a non-clustered AWX. +`CONTROL_PLANE_NODE_COUNT` is configurable and defaults to 1, effectively a non-clustered AWX. Note that you may see multiple messages of the form `2021-03-04 20:11:47,666 WARNING [-] awx.main.wsbroadcast Connection from awx_2 to awx_5 failed: 'Cannot connect to host awx_5:8013 ssl:False [Name or service not known]'.`. This can happen when you bring up a cluster of many nodes, say 10, then you bring up a cluster of less nodes, say 3. In this example, there will be 7 `Instance` records in the database that represent AWX instances. The AWX development environment mimics the VM deployment (vs. kubernetes) and expects the missing nodes to be brought back to healthy by the admin. The warning message you are seeing is all of the AWX nodes trying to connect the websocket backplane. You can manually delete the `Instance` records from the database i.e. `Instance.objects.get(hostname='awx_9').delete()` to stop the warnings. diff --git a/tools/docker-compose/ansible/roles/sources/defaults/main.yml b/tools/docker-compose/ansible/roles/sources/defaults/main.yml index b64afc5012..62d90ca4b9 100644 --- a/tools/docker-compose/ansible/roles/sources/defaults/main.yml +++ b/tools/docker-compose/ansible/roles/sources/defaults/main.yml @@ -5,5 +5,5 @@ awx_image: 'quay.io/ansible/awx_devel' pg_port: 5432 pg_username: 'awx' pg_database: 'awx' -cluster_node_count: 1 +control_plane_node_count: 1 minikube_container_group: false diff --git a/tools/docker-compose/ansible/roles/sources/tasks/main.yml b/tools/docker-compose/ansible/roles/sources/tasks/main.yml index 9cb959441e..57bc4ed922 100644 --- a/tools/docker-compose/ansible/roles/sources/tasks/main.yml +++ b/tools/docker-compose/ansible/roles/sources/tasks/main.yml @@ -1,10 +1,13 @@ --- -- name: Create _sources directory +- name: Create _sources directories file: - path: "{{ sources_dest }}/secrets" + path: "{{ sources_dest }}/{{ item }}" state: 'directory' mode: '0700' + loop: + - secrets + - receptor - name: Detect secrets stat: @@ -82,8 +85,15 @@ dest: "{{ sources_dest }}/{{ compose_name }}" mode: '0600' -- name: Render Docker-Compose (Receptor) +- name: Render Receptor Hop Config template: - src: docker-receptor.yml.j2 - dest: "{{ sources_dest }}/docker-receptor.yml" + src: "receptor-hop.conf.j2" + dest: "{{ sources_dest }}/receptor/receptor-hop.conf" mode: '0600' + +- name: Render Receptor Worker Config(s) + template: + src: "receptor-worker.conf.j2" + dest: "{{ sources_dest }}/receptor/receptor-worker-{{ item }}.conf" + mode: '0600' + with_sequence: start=1 end={{ execution_node_count }} diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index f448384020..c24e7e85f6 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -1,7 +1,7 @@ --- version: '2.1' services: -{% for i in range(cluster_node_count|int) %} +{% for i in range(control_plane_node_count|int) %} {% set container_postfix = loop.index %} {% set awx_sdb_port_start = 7899 + (loop.index0*1000) | int %} {% set awx_sdb_port_end = 7999 + (loop.index0*1000) | int %} @@ -27,7 +27,6 @@ services: links: - postgres - redis_{{ container_postfix }} - - execution_node_1 working_dir: "/awx_devel" volumes: - "../../../:/awx_devel" @@ -41,12 +40,11 @@ services: - "/sys/fs/cgroup:/sys/fs/cgroup" - "~/.kube/config:/var/lib/awx/.kube/config" - "redis_socket_{{ container_postfix }}:/var/run/redis/:rw" - - "execution_node_1:/var/run/receptor/" privileged: true tty: true ports: - "{{ awx_sdb_port_start }}-{{ awx_sdb_port_end }}:{{ awx_sdb_port_start }}-{{ awx_sdb_port_end }}" # sdb-listen -{% if cluster_node_count|int == 1 %} +{% if control_plane_node_count|int == 1 %} - "6899:6899" - "8080:8080" # unused but mapped for debugging - "8888:8888" # jupyter notebook @@ -63,17 +61,7 @@ services: entrypoint: ["redis-server"] command: ["/usr/local/etc/redis/redis.conf"] {% endfor %} - execution_node_1: - image: quay.io/awx/awx_devel:devel - user: "{{ ansible_user_uid }}" - container_name: tools_execution_node_1 - hostname: execution_node_1 - command: 'receptor --config /etc/receptor/awx-1-receptor-standalone.conf' - volumes: - - "../../docker-compose:/etc/receptor" - - "/sys/fs/cgroup:/sys/fs/cgroup" - privileged: true -{% if cluster_node_count|int > 1 %} +{% if control_plane_node_count|int > 1 %} haproxy: image: haproxy user: "{{ ansible_user_uid }}" @@ -84,7 +72,7 @@ services: - "8043:8043" - "1936:1936" depends_on: - {% for i in range(cluster_node_count|int) -%} + {% for i in range(control_plane_node_count|int) -%} {% set container_postfix = loop.index %} - "awx_{{ container_postfix }}" {% endfor %} @@ -105,13 +93,40 @@ services: POSTGRES_PASSWORD: {{ pg_password }} volumes: - "awx_db:/var/lib/postgresql/data" +{% if execution_node_count|int > 0 %} + receptor-hop: + image: quay.io/project-receptor/receptor:latest + user: root + container_name: tools_receptor_hop + hostname: receptor-hop + command: 'receptor --config /etc/receptor/receptor.conf' + links: + - awx_1 + ports: + - "5555:5555" + volumes: + - "../../docker-compose/_sources/receptor/receptor-hop.conf:/etc/receptor/receptor.conf" + {% for i in range(execution_node_count|int) -%} + receptor-{{ loop.index }}: + image: quay.io/awx/awx_devel:devel + user: "{{ ansible_user_uid }}" + container_name: tools_receptor_{{ loop.index }} + hostname: receptor-1 + command: 'receptor --config /etc/receptor/receptor.conf' + links: + - receptor-hop + volumes: + - "../../docker-compose/_sources/receptor/receptor-worker-{{ loop.index }}.conf:/etc/receptor/receptor.conf" + - "/sys/fs/cgroup:/sys/fs/cgroup" + privileged: true + {% endfor %} +{% endif %} + volumes: awx_db: name: tools_awx_db -{% for i in range(cluster_node_count|int) -%} +{% for i in range(control_plane_node_count|int) -%} {% set container_postfix = loop.index %} - execution_node_1: - name: tools_execution_node_1 redis_socket_{{ container_postfix }}: name: tools_redis_socket_{{ container_postfix }} {% endfor -%} diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-receptor.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-receptor.yml.j2 deleted file mode 100644 index 574349cf43..0000000000 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-receptor.yml.j2 +++ /dev/null @@ -1,66 +0,0 @@ -# -# a simple receptor network with a single control plane node and one hop -# to nodes where receptor + runner are installed -# -# -# -# ^-------------- -# ^-------------- -# ^-------------- -# ---- -version: '2.1' -services: - awx_1: - environment: - RECEPTOR_MUTUAL_TLS: awx receptor-hop receptor-1 receptor-2 receptor-3 - volumes: - - "./docker-compose-cluster:/etc/receptor" - receptor-hop: - image: quay.io/project-receptor/receptor:latest - user: root - container_name: tools_receptor_hop - hostname: receptor-hop - command: 'receptor --config /etc/receptor/awx-hop-receptor.conf' - links: - - awx_1 - ports: - - "5555:5555" - volumes: - - "../../docker-compose-cluster:/etc/receptor" - receptor-1: - image: quay.io/awx/awx_devel:devel - user: "{{ ansible_user_uid }}" - container_name: tools_receptor_1 - hostname: receptor-1 - command: 'receptor --config /etc/receptor/awx-1-receptor.conf' - links: - - receptor-hop - volumes: - - "../../docker-compose-cluster:/etc/receptor" - - "/sys/fs/cgroup:/sys/fs/cgroup" - privileged: true - receptor-2: - image: quay.io/awx/awx_devel:devel - user: "{{ ansible_user_uid }}" - container_name: tools_receptor_2 - hostname: receptor-2 - command: 'receptor --config /etc/receptor/awx-2-receptor.conf' - links: - - receptor-hop - volumes: - - "../../docker-compose-cluster:/etc/receptor" - - "/sys/fs/cgroup:/sys/fs/cgroup" - privileged: true - receptor-3: - image: quay.io/awx/awx_devel:devel - user: "{{ ansible_user_uid }}" - container_name: tools_receptor_3 - hostname: receptor-3 - command: 'receptor --config /etc/receptor/awx-3-receptor.conf' - links: - - receptor-hop - volumes: - - "../../docker-compose-cluster:/etc/receptor" - - "/sys/fs/cgroup:/sys/fs/cgroup" - privileged: true diff --git a/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2 b/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2 index 33837ded1f..1def668189 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2 @@ -29,7 +29,7 @@ backend nodes http-request set-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto https if { ssl_fc } option httpchk HEAD / HTTP/1.1\r\nHost:localhost -{% for i in range(cluster_node_count|int) %} +{% for i in range(control_plane_node_count|int) %} {% set container_postfix = loop.index %} server tools_awx_{{ container_postfix }} tools_awx_{{ container_postfix }}:8013 check {% endfor %} @@ -37,7 +37,7 @@ backend nodes backend nodes_ssl mode tcp balance roundrobin -{% for i in range(cluster_node_count|int) %} +{% for i in range(control_plane_node_count|int) %} {% set container_postfix = loop.index %} server tools_awx_{{ container_postfix }} tools_awx_{{ container_postfix }}:8043 check {% endfor %} diff --git a/tools/docker-compose/ansible/roles/sources/templates/receptor-hop.conf.j2 b/tools/docker-compose/ansible/roles/sources/templates/receptor-hop.conf.j2 new file mode 100644 index 0000000000..69fd33c3ca --- /dev/null +++ b/tools/docker-compose/ansible/roles/sources/templates/receptor-hop.conf.j2 @@ -0,0 +1,12 @@ +--- +- node: + id: receptor-hop + +- log-level: info + +- tcp-peer: + address: awx_1:4444 + redial: true + +- tcp-listener: + port: 5555 diff --git a/tools/docker-compose-cluster/awx-1-receptor.conf b/tools/docker-compose/ansible/roles/sources/templates/receptor-worker.conf.j2 similarity index 53% rename from tools/docker-compose-cluster/awx-1-receptor.conf rename to tools/docker-compose/ansible/roles/sources/templates/receptor-worker.conf.j2 index 82cd21471b..ef31f4d443 100644 --- a/tools/docker-compose-cluster/awx-1-receptor.conf +++ b/tools/docker-compose/ansible/roles/sources/templates/receptor-worker.conf.j2 @@ -1,19 +1,12 @@ --- - node: - id: receptor-1 + id: receptor-{{ item }} - log-level: info -#- tls-client: -# name: mutual-tls-client -# cert: /etc/receptor/certs/receptor-1.crt -# key: /etc/receptor/certs/receptor-1.key -# rootcas: /etc/receptor/certs/ca.crt - - tcp-peer: address: tools_receptor_hop:5555 redial: true - #tls: mutual-tls-client - work-command: worktype: ansible-runner