From f6a71e770db69ce4d51e04400b3aab21c0110892 Mon Sep 17 00:00:00 2001 From: Marcelo Moreira de Mello Date: Tue, 13 Jul 2021 00:36:19 -0400 Subject: [PATCH] Incorporates Minikube to devel environment --- .gitignore | 1 + Makefile | 18 ++++- tools/docker-compose-minikube/deploy.yml | 6 ++ .../minikube/defaults/main.yml | 13 +++ .../minikube/tasks/main.yml | 81 +++++++++++++++++++ .../templates/bootstrap_minikube.py.j2 | 42 ++++++++++ .../minikube/templates/rbac.yml.j2 | 38 +++++++++ tools/docker-compose/README.md | 21 +++++ .../ansible/roles/sources/defaults/main.yml | 1 + .../sources/templates/docker-compose.yml.j2 | 10 +++ tools/docker-compose/bootstrap_development.sh | 5 ++ tools/docker-compose/receptor.conf | 10 ++- 12 files changed, 241 insertions(+), 5 deletions(-) create mode 100644 tools/docker-compose-minikube/deploy.yml create mode 100644 tools/docker-compose-minikube/minikube/defaults/main.yml create mode 100644 tools/docker-compose-minikube/minikube/tasks/main.yml create mode 100644 tools/docker-compose-minikube/minikube/templates/bootstrap_minikube.py.j2 create mode 100644 tools/docker-compose-minikube/minikube/templates/rbac.yml.j2 diff --git a/.gitignore b/.gitignore index 3b40525d12..48b41ea40e 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ tools/docker-compose/Dockerfile tools/docker-compose/_build tools/docker-compose/_sources tools/docker-compose/overrides/ +tools/docker-compose-minikube/_sources # Tower setup playbook testing setup/test/roles/postgresql diff --git a/Makefile b/Makefile index e495485b0f..0526a6c65f 100644 --- a/Makefile +++ b/Makefile @@ -467,12 +467,19 @@ awx/projects: COMPOSE_UP_OPTS ?= CLUSTER_NODE_COUNT ?= 1 +MINIKUBE_CONTAINER_GROUP ?= false docker-compose-sources: .git/hooks/pre-commit + @if [ $(MINIKUBE_CONTAINER_GROUP) ]; then\ + ansible-playbook -i tools/docker-compose/inventory tools/docker-compose-minikube/deploy.yml; \ + fi; + ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \ -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \ -e awx_image_tag=$(COMPOSE_TAG) \ - -e cluster_node_count=$(CLUSTER_NODE_COUNT) + -e cluster_node_count=$(CLUSTER_NODE_COUNT) \ + -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP) + docker-compose: docker-auth awx/projects docker-compose-sources docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up @@ -498,6 +505,10 @@ detect-schema-change: genschema docker-compose-clean: awx/projects docker-compose -f tools/docker-compose/_sources/docker-compose.yml rm -sf +docker-compose-container-group-clean: + tools/docker-compose-minikube/_sources/minikube delete + rm -rf tools/docker-compose-minikube/_sources/ + # Base development image build docker-compose-build: ansible-playbook tools/ansible/dockerfile.yml -e build_dev=True @@ -509,7 +520,7 @@ docker-clean: $(foreach container_id,$(shell docker ps -f name=tools_awx -aq),docker stop $(container_id); docker rm -f $(container_id);) docker images | grep "awx_devel" | awk '{print $$1 ":" $$2}' | xargs docker rmi -docker-clean-volumes: docker-compose-clean +docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean docker volume rm tools_awx_db docker-refresh: docker-clean docker-compose @@ -524,6 +535,9 @@ docker-compose-cluster-elk: docker-auth awx/projects docker-compose-sources prometheus: docker run -u0 --net=tools_default --link=`docker ps | egrep -o "tools_awx(_run)?_([^ ]+)?"`:awxweb --volume `pwd`/tools/prometheus:/prometheus --name prometheus -d -p 0.0.0.0:9090:9090 prom/prometheus --web.enable-lifecycle --config.file=/prometheus/prometheus.yml +docker-compose-container-group: + MINIKUBE_CONTAINER_GROUP=true make docker-compose + clean-elk: docker stop tools_kibana_1 docker stop tools_logstash_1 diff --git a/tools/docker-compose-minikube/deploy.yml b/tools/docker-compose-minikube/deploy.yml new file mode 100644 index 0000000000..6425a2c52a --- /dev/null +++ b/tools/docker-compose-minikube/deploy.yml @@ -0,0 +1,6 @@ +--- +- name: Deploy Minikube and connect with AWX + hosts: localhost + gather_facts: true + roles: + - {role: minikube} diff --git a/tools/docker-compose-minikube/minikube/defaults/main.yml b/tools/docker-compose-minikube/minikube/defaults/main.yml new file mode 100644 index 0000000000..42ee8bdbe7 --- /dev/null +++ b/tools/docker-compose-minikube/minikube/defaults/main.yml @@ -0,0 +1,13 @@ +--- +sources_dest: '_sources' +driver: 'docker' + +minikube_url_linux: 'https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64' +minikube_url_macos: 'https://storage.googleapis.com/minikube/releases/latest/minikube-darwin-amd64' + +kubectl_url_linux: 'https://dl.k8s.io/release/v1.21.0/bin/linux/amd64/kubectl' +kubectl_url_macos: 'https://dl.k8s.io/release/v1.21.0/bin/darwin/amd64/kubectl' + +# Service Account Name +minikube_service_account_name: 'awx-devel' +minikube_service_account_namespace: 'default' diff --git a/tools/docker-compose-minikube/minikube/tasks/main.yml b/tools/docker-compose-minikube/minikube/tasks/main.yml new file mode 100644 index 0000000000..aa2c4c8c8d --- /dev/null +++ b/tools/docker-compose-minikube/minikube/tasks/main.yml @@ -0,0 +1,81 @@ +--- +- name: Create _sources directory + file: + path: "{{ sources_dest }}" + state: 'directory' + mode: '0700' + +# Linux block +- block: + - name: Download Minikube + get_url: + url: "{{ minikube_url_linux }}" + dest: "{{ sources_dest }}/minikube" + mode: 0755 + + - name: Download Kubectl + get_url: + url: "{{ kubectl_url_linux }}" + dest: "{{ sources_dest }}/kubectl" + mode: 0755 + when: + - ansible_architecture == "x86_64" + - ansible_system == "Linux" + +# MacOS block +- block: + - name: Download Minikube + get_url: + url: "{{ minikube_url_macos }}" + dest: "{{ sources_dest }}/minikube" + mode: 0755 + + - name: Download Kubectl + get_url: + url: "{{ kubectl_url_macos }}" + dest: "{{ sources_dest }}/kubectl" + mode: 0755 + when: + - ansible_architecture == "x86_64" + - ansible_system == "Darwin" + +- name: Starting Minikube + shell: "{{ sources_dest }}/minikube start --driver={{ driver }} --install-addons=true --addons=ingress" + +- name: Create ServiceAccount and clusterRoleBinding + k8s: + apply: true + definition: "{{ lookup('template', 'rbac.yml.j2') }}" + +- name: Retrieve serviceAccount secret name + k8s_info: + kind: ServiceAccount + namespace: '{{ minikube_service_account_namespace }}' + name: '{{ minikube_service_account_name }}' + register: service_account + +- name: Register serviceAccount secret name + set_fact: + _service_account_secret_name: '{{ service_account["resources"][0]["secrets"][0]["name"] }}' + when: + - service_account["resources"][0]["secrets"] | length + - '"name" in service_account["resources"][0]["secrets"][0]' + +- name: Retrieve bearer_token from serviceAccount secret + k8s_info: + kind: Secret + namespace: '{{ minikube_service_account_namespace }}' + name: '{{ _service_account_secret_name }}' + register: _service_account_secret + +- name: Load Minikube Bearer Token + set_fact: + service_account_token: '{{ _service_account_secret["resources"][0]["data"]["token"] | b64decode }}' + when: + - _service_account_secret["resources"][0]["data"] | length + +- name: Render minikube credential JSON template + template: + src: bootstrap_minikube.py.j2 + dest: "{{ sources_dest }}/bootstrap_minikube.py" + mode: '0600' diff --git a/tools/docker-compose-minikube/minikube/templates/bootstrap_minikube.py.j2 b/tools/docker-compose-minikube/minikube/templates/bootstrap_minikube.py.j2 new file mode 100644 index 0000000000..861ba5e23a --- /dev/null +++ b/tools/docker-compose-minikube/minikube/templates/bootstrap_minikube.py.j2 @@ -0,0 +1,42 @@ +# Create Openshift/Kubernetes credential for Minikube +# This script gets called by the bootstrap_development process +# awx-manage shell_plus --quiet < bootstrap_minikube.py + +from awx.main.utils.encryption import encrypt_field + +NAME = 'Minikube' + +POD_SPEC = """apiVersion: v1 +kind: Pod +metadata: + namespace: {{ minikube_service_account_namespace }} +spec: + containers: + - image: 'quay.io/ansible/awx-ee:devel' + name: worker + args: + - ansible-runner + - worker + - '--private-data-dir=/runner'""" + +# Creates Minikube credential +if not Credential.objects.filter(name=NAME).count(): + cred = Credential() + cred.name = NAME + cred.credential_type = CredentialType.objects.get(name='OpenShift or Kubernetes API Bearer Token') + cred.description = 'Minikube Devel' + cred.inputs['host'] = 'https://minikube:8443' + cred.inputs['verify_ssl'] = False + cred.inputs['bearer_token'] = '{{ service_account_token }}' + encrypt_field(cred, 'bearer_token', secret_key=settings.SECRET_KEY) + cred.save() + +# Create Container Group for Minikube +if not InstanceGroup.objects.filter(name=NAME).count(): + ccgrp = InstanceGroup() + ccgrp.name = NAME + ccgrp.credential = cred + ccgrp.pod_spec_override = POD_SPEC + ccgrp.is_container_group = True + ccgrp.save() + diff --git a/tools/docker-compose-minikube/minikube/templates/rbac.yml.j2 b/tools/docker-compose-minikube/minikube/templates/rbac.yml.j2 new file mode 100644 index 0000000000..642e2d94ca --- /dev/null +++ b/tools/docker-compose-minikube/minikube/templates/rbac.yml.j2 @@ -0,0 +1,38 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ minikube_service_account_name }} + namespace: {{ minikube_service_account_namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ minikube_service_account_name }} + namespace: {{ minikube_service_account_namespace }} +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] +- apiGroups: [""] + resources: ["pods/attach"] + verbs: ["create"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ minikube_service_account_name }} + namespace: {{ minikube_service_account_namespace }} +subjects: +- kind: ServiceAccount + name: {{ minikube_service_account_name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ minikube_service_account_name }} diff --git a/tools/docker-compose/README.md b/tools/docker-compose/README.md index 31bd5d0978..f506309106 100644 --- a/tools/docker-compose/README.md +++ b/tools/docker-compose/README.md @@ -213,3 +213,24 @@ Certain features or bugs are only applicable when running a cluster of AWX nodes `CLUSTER_NODE_COUNT` is configurable and defaults to 1, effectively a non-clustered AWX. Note that you may see multiple messages of the form `2021-03-04 20:11:47,666 WARNING [-] awx.main.wsbroadcast Connection from awx_2 to awx_5 failed: 'Cannot connect to host awx_5:8013 ssl:False [Name or service not known]'.`. This can happen when you bring up a cluster of many nodes, say 10, then you bring up a cluster of less nodes, say 3. In this example, there will be 7 `Instance` records in the database that represent AWX instances. The AWX development environment mimics the VM deployment (vs. kubernetes) and expects the missing nodes to be brought back to healthy by the admin. The warning message you are seeing is all of the AWX nodes trying to connect the websocket backplane. You can manually delete the `Instance` records from the database i.e. `Instance.objects.get(hostname='awx_9').delete()` to stop the warnings. + +### Start with Minikube + +To bring up a 1 node AWX + minikube that is accessible from AWX run the following. + +```bash +(host)$ make docker-compose-container-group +``` + +Alternatively, you can set the env var `MINIKUBE_CONTAINER_GROUP=true` to use the default dev env bring up. his way you can use other env flags like the cluster node count. + + +```bash +(host)$ MINIKUBE_CONTAINER_GROUP=true make docker-compose +``` + +If you want to clean all things once your are done, you can do: + +```bash +(host)$ make docker-compose-container-group-clean +``` \ No newline at end of file diff --git a/tools/docker-compose/ansible/roles/sources/defaults/main.yml b/tools/docker-compose/ansible/roles/sources/defaults/main.yml index ed0cfee862..b64afc5012 100644 --- a/tools/docker-compose/ansible/roles/sources/defaults/main.yml +++ b/tools/docker-compose/ansible/roles/sources/defaults/main.yml @@ -6,3 +6,4 @@ pg_port: 5432 pg_username: 'awx' pg_database: 'awx' cluster_node_count: 1 +minikube_container_group: false diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 8f884f9150..a445fdb1fd 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -20,6 +20,9 @@ services: RECEPTORCTL_SOCKET: /var/run/receptor/receptor.sock {% if loop.index == 1 %} RUN_MIGRATIONS: 1 +{% endif %} +{% if minikube_container_group|bool %} + MINIKUBE_CONTAINER_GROUP: "true" {% endif %} links: - postgres @@ -47,6 +50,7 @@ services: - "8888:8888" # jupyter notebook - "8013:8013" # http - "8043:8043" # https + - "2222:2222" # receptor foo node {% endif %} redis_{{ container_postfix }}: image: redis:latest @@ -98,3 +102,9 @@ volumes: redis_socket_{{ container_postfix }}: name: tools_redis_socket_{{ container_postfix }} {% endfor -%} +{% if minikube_container_group|bool %} +networks: + default: + external: + name: minikube +{% endif %} diff --git a/tools/docker-compose/bootstrap_development.sh b/tools/docker-compose/bootstrap_development.sh index 1b2b1a29e9..7b5d26b4cf 100755 --- a/tools/docker-compose/bootstrap_development.sh +++ b/tools/docker-compose/bootstrap_development.sh @@ -34,3 +34,8 @@ awx-manage register_default_execution_environments mkdir -p /awx_devel/awx/public/static mkdir -p /awx_devel/awx/ui/static mkdir -p /awx_devel/awx/ui_next/build/static + +# Create resource entries when using Minikube +if [[ -n "$MINIKUBE_CONTAINER_GROUP" ]]; then + awx-manage shell_plus --quiet < /awx_devel/tools/docker-compose-minikube/_sources/bootstrap_minikube.py +fi diff --git a/tools/docker-compose/receptor.conf b/tools/docker-compose/receptor.conf index 7192be28dc..5c2d94d51c 100644 --- a/tools/docker-compose/receptor.conf +++ b/tools/docker-compose/receptor.conf @@ -1,12 +1,16 @@ --- -- log-level: info +- node: + id: foo + +- log-level: debug + +- tcp-listener: + port: 2222 - control-service: service: control filename: /var/run/receptor/receptor.sock -- local-only: - - work-command: worktype: local command: ansible-runner