Merge pull request #10636 from tchellomello/minikube-devel-hack

Incorporates Minikube to devel environment SUMMARY Incorporates Minikube to devel environment This PR allows creating a smooth devel environment deploying Minikube connected to the AWX devel network interface using Docker. Furthermore, both CredentialType and Container Group gets created automatically. Steps performed by the target make docker-compose-container-group Downloads Minikube (currently supported Linux and MacOS) Downloads kubectl (currently supported Linux and MacOS) Starts Minikube using docker as backend Creates a Service Account, Role and RoleBinding on Minikube Exports Minikube ServiceAccount token to be used on AWX Credential Creates Minikube AWX Credential Creates Container Group to use Minikube backend Starts regular AWX devel instances and connected with Minikube network ISSUE TYPE Feature Pull Request COMPONENT NAME API AWX VERSION devel ADDITIONAL INFORMATION Quick step-by-step to use this PR: $ make docker-compose-container-group-clean # this will remove your database and any old Minikube already deployed $ make docker-compose-container-group Verifying some work: $ kubectl get serviceaccount awx-devel NAME SECRETS AGE awx-devel 1 130m $ kubectl get roles NAME CREATED AT awx-devel 2021-07-15T00:47:37Z $ kubectl get rolebindings NAME ROLE AGE awx-devel Role/awx-devel 131m Credential Container Instance Group After updating a Job Template and assigning to the Container Instance Group kubectl get pods -w NAME READY STATUS RESTARTS AGE automation-job-6-w6rmh 0/1 Pending 0 0s automation-job-6-w6rmh 0/1 Pending 0 0s automation-job-6-w6rmh 0/1 ContainerCreating 0 0s automation-job-6-w6rmh 1/1 Running 0 5s automation-job-6-w6rmh 1/1 Terminating 0 6s Reviewed-by: Shane McDonald <me@shanemcd.com> Reviewed-by: Elijah DeLee <kdelee@redhat.com> Reviewed-by: Marcelo Moreira de Mello <tchello.mello@gmail.com> Reviewed-by: Sarabraj Singh <singh.sarabraj@gmail.com>
2026-01-16 12:20:45 -03:30 · 2021-07-15 16:46:26 +00:00 · 2021-07-15 16:46:26 +00:00 · 9ed1f3bc0f
commit 9ed1f3bc0f
parent 4b81df2ab4 b63313a08b
12 changed files with 243 additions and 5 deletions
--- a/.gitignore
+++ b/.gitignore
@ -41,6 +41,7 @@ tools/docker-compose/Dockerfile
 tools/docker-compose/_build
 tools/docker-compose/_sources
 tools/docker-compose/overrides/
+tools/docker-compose-minikube/_sources

 # Tower setup playbook testing
 setup/test/roles/postgresql
--- a/18
+++ b/18
@ -467,12 +467,19 @@ awx/projects:

 COMPOSE_UP_OPTS ?=
 CLUSTER_NODE_COUNT ?= 1
+MINIKUBE_CONTAINER_GROUP ?= false

 docker-compose-sources: .git/hooks/pre-commit
+	@if [ $(MINIKUBE_CONTAINER_GROUP) ]; then\
+	    ansible-playbook -i tools/docker-compose/inventory tools/docker-compose-minikube/deploy.yml; \
+	fi;
+
 	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \
 	    -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \
 	    -e awx_image_tag=$(COMPOSE_TAG) \
-	    -e cluster_node_count=$(CLUSTER_NODE_COUNT)
+	    -e cluster_node_count=$(CLUSTER_NODE_COUNT) \
+	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP)
+

 docker-compose: docker-auth awx/projects docker-compose-sources
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up
@ -498,6 +505,10 @@ detect-schema-change: genschema
 docker-compose-clean: awx/projects
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml rm -sf

+docker-compose-container-group-clean:
+	tools/docker-compose-minikube/_sources/minikube delete
+	rm -rf tools/docker-compose-minikube/_sources/
+
 # Base development image build
 docker-compose-build:
 	ansible-playbook tools/ansible/dockerfile.yml -e build_dev=True
@ -509,7 +520,7 @@ docker-clean:
 	$(foreach container_id,$(shell docker ps -f name=tools_awx -aq),docker stop $(container_id); docker rm -f $(container_id);)
 	docker images | grep "awx_devel" | awk '{print $$1 ":" $$2}' | xargs docker rmi

-docker-clean-volumes: docker-compose-clean
+docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
 	docker volume rm tools_awx_db

 docker-refresh: docker-clean docker-compose
@ -524,6 +535,9 @@ docker-compose-cluster-elk: docker-auth awx/projects docker-compose-sources
 prometheus:
 	docker run -u0 --net=tools_default --link=`docker ps | egrep -o "tools_awx(_run)?_([^ ]+)?"`:awxweb --volume `pwd`/tools/prometheus:/prometheus --name prometheus -d -p 0.0.0.0:9090:9090 prom/prometheus --web.enable-lifecycle --config.file=/prometheus/prometheus.yml

+docker-compose-container-group:
+	MINIKUBE_CONTAINER_GROUP=true make docker-compose
+
 clean-elk:
 	docker stop tools_kibana_1
 	docker stop tools_logstash_1
--- a/tools/docker-compose-minikube/deploy.yml
+++ b/tools/docker-compose-minikube/deploy.yml
@ -0,0 +1,6 @@
+---
+- name: Deploy Minikube and connect with AWX
+  hosts: localhost
+  gather_facts: true
+  roles:
+    - {role: minikube}
--- a/tools/docker-compose-minikube/minikube/defaults/main.yml
+++ b/tools/docker-compose-minikube/minikube/defaults/main.yml
@ -0,0 +1,13 @@
+---
+sources_dest: '_sources'
+driver: 'docker'
+
+minikube_url_linux: 'https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64'
+minikube_url_macos: 'https://storage.googleapis.com/minikube/releases/latest/minikube-darwin-amd64'
+
+kubectl_url_linux: 'https://dl.k8s.io/release/v1.21.0/bin/linux/amd64/kubectl'
+kubectl_url_macos: 'https://dl.k8s.io/release/v1.21.0/bin/darwin/amd64/kubectl'
+
+# Service Account Name
+minikube_service_account_name: 'awx-devel'
+minikube_service_account_namespace: 'default'
--- a/tools/docker-compose-minikube/minikube/tasks/main.yml
+++ b/tools/docker-compose-minikube/minikube/tasks/main.yml
@ -0,0 +1,81 @@
+---
+- name: Create _sources directory
+  file:
+    path: "{{ sources_dest }}"
+    state: 'directory'
+    mode: '0700'
+
+# Linux block
+- block:
+    - name: Download Minikube
+      get_url:
+        url: "{{ minikube_url_linux }}"
+        dest: "{{ sources_dest }}/minikube"
+        mode: 0755
+
+    - name: Download Kubectl
+      get_url:
+        url: "{{ kubectl_url_linux }}"
+        dest: "{{ sources_dest }}/kubectl"
+        mode: 0755
+  when:
+    - ansible_architecture == "x86_64"
+    - ansible_system == "Linux"
+
+# MacOS block
+- block:
+    - name: Download Minikube
+      get_url:
+        url: "{{ minikube_url_macos }}"
+        dest: "{{ sources_dest }}/minikube"
+        mode: 0755
+
+    - name: Download Kubectl
+      get_url:
+        url: "{{ kubectl_url_macos }}"
+        dest: "{{ sources_dest }}/kubectl"
+        mode: 0755
+  when:
+    - ansible_architecture == "x86_64"
+    - ansible_system == "Darwin"
+
+- name: Starting Minikube
+  shell: "{{ sources_dest }}/minikube start --driver={{ driver }} --install-addons=true --addons=ingress"
+
+- name: Create ServiceAccount and clusterRoleBinding
+  k8s:
+    apply: true
+    definition: "{{ lookup('template', 'rbac.yml.j2') }}"
+
+- name: Retrieve serviceAccount secret name
+  k8s_info:
+    kind: ServiceAccount
+    namespace: '{{ minikube_service_account_namespace }}'
+    name: '{{ minikube_service_account_name }}'
+  register: service_account
+
+- name: Register serviceAccount secret name
+  set_fact:
+    _service_account_secret_name: '{{ service_account["resources"][0]["secrets"][0]["name"] }}'
+  when:
+    - service_account["resources"][0]["secrets"] | length
+    - '"name" in service_account["resources"][0]["secrets"][0]'
+
+- name: Retrieve bearer_token from serviceAccount secret
+  k8s_info:
+    kind: Secret
+    namespace: '{{ minikube_service_account_namespace }}'
+    name: '{{ _service_account_secret_name }}'
+  register: _service_account_secret
+
+- name: Load Minikube Bearer Token
+  set_fact:
+    service_account_token: '{{ _service_account_secret["resources"][0]["data"]["token"] | b64decode }}'
+  when:
+    - _service_account_secret["resources"][0]["data"] | length
+
+- name: Render minikube credential JSON template
+  template:
+    src: bootstrap_minikube.py.j2
+    dest: "{{ sources_dest }}/bootstrap_minikube.py"
+    mode: '0600'
--- a/tools/docker-compose-minikube/minikube/templates/bootstrap_minikube.py.j2
+++ b/tools/docker-compose-minikube/minikube/templates/bootstrap_minikube.py.j2
@ -0,0 +1,44 @@
+# Create Openshift/Kubernetes credential for Minikube
+# This script gets called by the bootstrap_development process
+# awx-manage shell < bootstrap_minikube.py
+
+from awx.main.utils.encryption import encrypt_field
+from awx.main.models import Credential, CredentialType, InstanceGroup
+from django.conf import settings
+
+NAME = 'Minikube'
+
+POD_SPEC = """apiVersion: v1
+kind: Pod
+metadata:
+  namespace: {{ minikube_service_account_namespace }}
+spec:
+  containers:
+    - image: 'quay.io/ansible/awx-ee:devel'
+      name: worker
+      args:
+        - ansible-runner
+        - worker
+        - '--private-data-dir=/runner'"""
+
+# Creates Minikube credential
+if not Credential.objects.filter(name=NAME).count():
+    cred = Credential()
+    cred.name = NAME
+    cred.credential_type = CredentialType.objects.get(name='OpenShift or Kubernetes API Bearer Token')
+    cred.description = 'Minikube Devel'
+    cred.inputs['host'] =  'https://minikube:8443'
+    cred.inputs['verify_ssl'] =  False
+    cred.inputs['bearer_token'] = '{{ service_account_token }}'
+    encrypt_field(cred, 'bearer_token', secret_key=settings.SECRET_KEY)
+    cred.save()
+
+# Create Container Group for Minikube
+if not InstanceGroup.objects.filter(name=NAME).count():
+    ccgrp = InstanceGroup()
+    ccgrp.name = NAME
+    ccgrp.credential = cred
+    ccgrp.pod_spec_override = POD_SPEC
+    ccgrp.is_container_group = True
+    ccgrp.save()
+
--- a/tools/docker-compose-minikube/minikube/templates/rbac.yml.j2
+++ b/tools/docker-compose-minikube/minikube/templates/rbac.yml.j2
@ -0,0 +1,38 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ minikube_service_account_name }}
+  namespace: {{ minikube_service_account_namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ minikube_service_account_name }}
+  namespace: {{ minikube_service_account_namespace }}
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["pods"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["pods/attach"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ minikube_service_account_name }}
+  namespace: {{ minikube_service_account_namespace }}
+subjects:
+- kind: ServiceAccount
+  name: {{ minikube_service_account_name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ minikube_service_account_name }}
--- a/tools/docker-compose/README.md
+++ b/tools/docker-compose/README.md
@ -213,3 +213,24 @@ Certain features or bugs are only applicable when running a cluster of AWX nodes
 `CLUSTER_NODE_COUNT` is configurable and defaults to 1, effectively a non-clustered AWX.

 Note that you may see multiple messages of the form `2021-03-04 20:11:47,666 WARNING  [-] awx.main.wsbroadcast Connection from awx_2 to awx_5 failed: 'Cannot connect to host awx_5:8013 ssl:False [Name or service not known]'.`. This can happen when you bring up a cluster of many nodes, say 10, then you bring up a cluster of less nodes, say 3. In this example, there will be 7 `Instance` records in the database that represent AWX instances. The AWX development environment mimics the VM deployment (vs. kubernetes) and expects the missing nodes to be brought back to healthy by the admin. The warning message you are seeing is all of the AWX nodes trying to connect the websocket backplane. You can manually delete the `Instance` records from the database i.e. `Instance.objects.get(hostname='awx_9').delete()` to stop the warnings.
+
+### Start with Minikube
+
+To bring up a 1 node AWX + minikube that is accessible from AWX run the following.
+
+```bash
+(host)$ make docker-compose-container-group
+```
+
+Alternatively, you can set the env var `MINIKUBE_CONTAINER_GROUP=true` to use the default dev env bring up. his way you can use other env flags like the cluster node count.
+
+
+```bash
+(host)$ MINIKUBE_CONTAINER_GROUP=true make docker-compose
+```
+
+If you want to clean all things once your are done, you can do:
+
+```bash
+(host)$ make docker-compose-container-group-clean
+```
--- a/tools/docker-compose/ansible/roles/sources/defaults/main.yml
+++ b/tools/docker-compose/ansible/roles/sources/defaults/main.yml
@ -6,3 +6,4 @@ pg_port: 5432
 pg_username: 'awx'
 pg_database: 'awx'
 cluster_node_count: 1
+minikube_container_group: false
--- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2
+++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2
@ -20,6 +20,9 @@ services:
      RECEPTORCTL_SOCKET: /var/run/receptor/receptor.sock
 {% if loop.index == 1 %}
      RUN_MIGRATIONS: 1
+{% endif %}
+{% if minikube_container_group|bool %}
+      MINIKUBE_CONTAINER_GROUP: "true"
 {% endif %}
    links:
      - postgres
@ -47,6 +50,7 @@ services:
      - "8888:8888"  # jupyter notebook
      - "8013:8013"  # http
      - "8043:8043"  # https
+      - "2222:2222"  # receptor foo node
 {% endif %}
  redis_{{ container_postfix }}:
    image: redis:latest
@ -98,3 +102,9 @@ volumes:
  redis_socket_{{ container_postfix }}:
    name: tools_redis_socket_{{ container_postfix }}
 {% endfor -%}
+{% if minikube_container_group|bool %}
+networks:
+  default:
+    external:
+      name: minikube
+{% endif %}
--- a/tools/docker-compose/bootstrap_development.sh
+++ b/tools/docker-compose/bootstrap_development.sh
@ -34,3 +34,8 @@ awx-manage register_default_execution_environments
 mkdir -p /awx_devel/awx/public/static
 mkdir -p /awx_devel/awx/ui/static
 mkdir -p /awx_devel/awx/ui_next/build/static
+
+# Create resource entries when using Minikube
+if [[ -n "$MINIKUBE_CONTAINER_GROUP" ]]; then
+    awx-manage shell < /awx_devel/tools/docker-compose-minikube/_sources/bootstrap_minikube.py
+fi
--- a/tools/docker-compose/receptor.conf
+++ b/tools/docker-compose/receptor.conf
@ -1,12 +1,16 @@
 ---
- log-level: info
+- node:
+    id: foo
+
+- log-level: debug
+
+- tcp-listener:
+    port: 2222

 - control-service:
    service: control
    filename: /var/run/receptor/receptor.sock

- local-only:
-
 - work-command:
    worktype: local
    command: ansible-runner