Incorporates Minikube to devel environment

.gitignore

@@ -41,6 +41,7 @@ tools/docker-compose/Dockerfile
 tools/docker-compose/_build
 tools/docker-compose/_sources
 tools/docker-compose/overrides/
+tools/docker-compose-minikube/_sources
 
 # Tower setup playbook testing
 setup/test/roles/postgresql

Makefile

@@ -467,12 +467,19 @@ awx/projects:
 
 COMPOSE_UP_OPTS ?=
 CLUSTER_NODE_COUNT ?= 1
+MINIKUBE_CONTAINER_GROUP ?= false
 
 docker-compose-sources: .git/hooks/pre-commit
+	@if [ $(MINIKUBE_CONTAINER_GROUP) ]; then\
+	    ansible-playbook -i tools/docker-compose/inventory tools/docker-compose-minikube/deploy.yml; \
+	fi;
+
 	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \
 	    -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \
 	    -e awx_image_tag=$(COMPOSE_TAG) \
-	    -e cluster_node_count=$(CLUSTER_NODE_COUNT)
+	    -e cluster_node_count=$(CLUSTER_NODE_COUNT) \
+	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP)
+
 
 docker-compose: docker-auth awx/projects docker-compose-sources
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_UP_OPTS) up
@@ -498,6 +505,10 @@ detect-schema-change: genschema
 docker-compose-clean: awx/projects
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml rm -sf
 
+docker-compose-container-group-clean:
+	tools/docker-compose-minikube/_sources/minikube delete
+	rm -rf tools/docker-compose-minikube/_sources/
+
 # Base development image build
 docker-compose-build:
 	ansible-playbook tools/ansible/dockerfile.yml -e build_dev=True
@@ -509,7 +520,7 @@ docker-clean:
 	$(foreach container_id,$(shell docker ps -f name=tools_awx -aq),docker stop $(container_id); docker rm -f $(container_id);)
 	docker images | grep "awx_devel" | awk '{print $$1 ":" $$2}' | xargs docker rmi
 
-docker-clean-volumes: docker-compose-clean
+docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
 	docker volume rm tools_awx_db
 
 docker-refresh: docker-clean docker-compose
@@ -524,6 +535,9 @@ docker-compose-cluster-elk: docker-auth awx/projects docker-compose-sources
 prometheus:
 	docker run -u0 --net=tools_default --link=`docker ps | egrep -o "tools_awx(_run)?_([^ ]+)?"`:awxweb --volume `pwd`/tools/prometheus:/prometheus --name prometheus -d -p 0.0.0.0:9090:9090 prom/prometheus --web.enable-lifecycle --config.file=/prometheus/prometheus.yml
 
+docker-compose-container-group:
+	MINIKUBE_CONTAINER_GROUP=true make docker-compose
+
 clean-elk:
 	docker stop tools_kibana_1
 	docker stop tools_logstash_1

tools/docker-compose-minikube/deploy.yml

@@ -0,0 +1,6 @@
+---
+- name: Deploy Minikube and connect with AWX
+  hosts: localhost
+  gather_facts: true
+  roles:
+    - {role: minikube}

tools/docker-compose-minikube/minikube/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+sources_dest: '_sources'
+driver: 'docker'
+
+minikube_url_linux: 'https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64'
+minikube_url_macos: 'https://storage.googleapis.com/minikube/releases/latest/minikube-darwin-amd64'
+
+kubectl_url_linux: 'https://dl.k8s.io/release/v1.21.0/bin/linux/amd64/kubectl'
+kubectl_url_macos: 'https://dl.k8s.io/release/v1.21.0/bin/darwin/amd64/kubectl'
+
+# Service Account Name
+minikube_service_account_name: 'awx-devel'
+minikube_service_account_namespace: 'default'

tools/docker-compose-minikube/minikube/tasks/main.yml

@@ -0,0 +1,81 @@
+---
+- name: Create _sources directory
+  file:
+    path: "{{ sources_dest }}"
+    state: 'directory'
+    mode: '0700'
+
+# Linux block
+- block:
+    - name: Download Minikube
+      get_url:
+        url: "{{ minikube_url_linux }}"
+        dest: "{{ sources_dest }}/minikube"
+        mode: 0755
+
+    - name: Download Kubectl
+      get_url:
+        url: "{{ kubectl_url_linux }}"
+        dest: "{{ sources_dest }}/kubectl"
+        mode: 0755
+  when:
+    - ansible_architecture == "x86_64"
+    - ansible_system == "Linux"
+
+# MacOS block
+- block:
+    - name: Download Minikube
+      get_url:
+        url: "{{ minikube_url_macos }}"
+        dest: "{{ sources_dest }}/minikube"
+        mode: 0755
+
+    - name: Download Kubectl
+      get_url:
+        url: "{{ kubectl_url_macos }}"
+        dest: "{{ sources_dest }}/kubectl"
+        mode: 0755
+  when:
+    - ansible_architecture == "x86_64"
+    - ansible_system == "Darwin"
+
+- name: Starting Minikube
+  shell: "{{ sources_dest }}/minikube start --driver={{ driver }} --install-addons=true --addons=ingress"
+
+- name: Create ServiceAccount and clusterRoleBinding
+  k8s:
+    apply: true
+    definition: "{{ lookup('template', 'rbac.yml.j2') }}"
+
+- name: Retrieve serviceAccount secret name
+  k8s_info:
+    kind: ServiceAccount
+    namespace: '{{ minikube_service_account_namespace }}'
+    name: '{{ minikube_service_account_name }}'
+  register: service_account
+
+- name: Register serviceAccount secret name
+  set_fact:
+    _service_account_secret_name: '{{ service_account["resources"][0]["secrets"][0]["name"] }}'
+  when:
+    - service_account["resources"][0]["secrets"] | length
+    - '"name" in service_account["resources"][0]["secrets"][0]'
+
+- name: Retrieve bearer_token from serviceAccount secret
+  k8s_info:
+    kind: Secret
+    namespace: '{{ minikube_service_account_namespace }}'
+    name: '{{ _service_account_secret_name }}'
+  register: _service_account_secret
+
+- name: Load Minikube Bearer Token
+  set_fact:
+    service_account_token: '{{ _service_account_secret["resources"][0]["data"]["token"] | b64decode }}'
+  when:
+    - _service_account_secret["resources"][0]["data"] | length
+
+- name: Render minikube credential JSON template
+  template:
+    src: bootstrap_minikube.py.j2
+    dest: "{{ sources_dest }}/bootstrap_minikube.py"
+    mode: '0600'

tools/docker-compose-minikube/minikube/templates/bootstrap_minikube.py.j2

@@ -0,0 +1,42 @@
+# Create Openshift/Kubernetes credential for Minikube
+# This script gets called by the bootstrap_development process
+# awx-manage shell_plus --quiet < bootstrap_minikube.py
+
+from awx.main.utils.encryption import encrypt_field
+
+NAME = 'Minikube'
+
+POD_SPEC = """apiVersion: v1
+kind: Pod
+metadata:
+  namespace: {{ minikube_service_account_namespace }}
+spec:
+  containers:
+    - image: 'quay.io/ansible/awx-ee:devel'
+      name: worker
+      args:
+        - ansible-runner
+        - worker
+        - '--private-data-dir=/runner'"""
+
+# Creates Minikube credential
+if not Credential.objects.filter(name=NAME).count():
+    cred = Credential()
+    cred.name = NAME
+    cred.credential_type = CredentialType.objects.get(name='OpenShift or Kubernetes API Bearer Token')
+    cred.description = 'Minikube Devel'
+    cred.inputs['host'] =  'https://minikube:8443'
+    cred.inputs['verify_ssl'] =  False
+    cred.inputs['bearer_token'] = '{{ service_account_token }}'
+    encrypt_field(cred, 'bearer_token', secret_key=settings.SECRET_KEY)
+    cred.save()
+
+# Create Container Group for Minikube
+if not InstanceGroup.objects.filter(name=NAME).count():
+    ccgrp = InstanceGroup()
+    ccgrp.name = NAME
+    ccgrp.credential = cred
+    ccgrp.pod_spec_override = POD_SPEC
+    ccgrp.is_container_group = True
+    ccgrp.save()
+

tools/docker-compose-minikube/minikube/templates/rbac.yml.j2

@@ -0,0 +1,38 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ minikube_service_account_name }}
+  namespace: {{ minikube_service_account_namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ minikube_service_account_name }}
+  namespace: {{ minikube_service_account_namespace }}
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["pods"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["pods/attach"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ minikube_service_account_name }}
+  namespace: {{ minikube_service_account_namespace }}
+subjects:
+- kind: ServiceAccount
+  name: {{ minikube_service_account_name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ minikube_service_account_name }}

tools/docker-compose/README.md

@@ -213,3 +213,24 @@ Certain features or bugs are only applicable when running a cluster of AWX nodes
 `CLUSTER_NODE_COUNT` is configurable and defaults to 1, effectively a non-clustered AWX.
 
 Note that you may see multiple messages of the form `2021-03-04 20:11:47,666 WARNING  [-] awx.main.wsbroadcast Connection from awx_2 to awx_5 failed: 'Cannot connect to host awx_5:8013 ssl:False [Name or service not known]'.`. This can happen when you bring up a cluster of many nodes, say 10, then you bring up a cluster of less nodes, say 3. In this example, there will be 7 `Instance` records in the database that represent AWX instances. The AWX development environment mimics the VM deployment (vs. kubernetes) and expects the missing nodes to be brought back to healthy by the admin. The warning message you are seeing is all of the AWX nodes trying to connect the websocket backplane. You can manually delete the `Instance` records from the database i.e. `Instance.objects.get(hostname='awx_9').delete()` to stop the warnings.
+
+### Start with Minikube
+
+To bring up a 1 node AWX + minikube that is accessible from AWX run the following.
+
+```bash
+(host)$ make docker-compose-container-group
+```
+
+Alternatively, you can set the env var `MINIKUBE_CONTAINER_GROUP=true` to use the default dev env bring up. his way you can use other env flags like the cluster node count.
+
+
+```bash
+(host)$ MINIKUBE_CONTAINER_GROUP=true make docker-compose
+```
+
+If you want to clean all things once your are done, you can do:
+
+```bash
+(host)$ make docker-compose-container-group-clean
+```

tools/docker-compose/ansible/roles/sources/defaults/main.yml

@@ -6,3 +6,4 @@ pg_port: 5432
 pg_username: 'awx'
 pg_database: 'awx'
 cluster_node_count: 1
+minikube_container_group: false

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -20,6 +20,9 @@ services:
       RECEPTORCTL_SOCKET: /var/run/receptor/receptor.sock
 {% if loop.index == 1 %}
       RUN_MIGRATIONS: 1
+{% endif %}
+{% if minikube_container_group|bool %}
+      MINIKUBE_CONTAINER_GROUP: "true"
 {% endif %}
     links:
       - postgres
@@ -47,6 +50,7 @@ services:
       - "8888:8888"  # jupyter notebook
       - "8013:8013"  # http
       - "8043:8043"  # https
+      - "2222:2222"  # receptor foo node
 {% endif %}
   redis_{{ container_postfix }}:
     image: redis:latest
@@ -98,3 +102,9 @@ volumes:
   redis_socket_{{ container_postfix }}:
     name: tools_redis_socket_{{ container_postfix }}
 {% endfor -%}
+{% if minikube_container_group|bool %}
+networks:
+  default:
+    external:
+      name: minikube
+{% endif %}

tools/docker-compose/bootstrap_development.sh

@@ -34,3 +34,8 @@ awx-manage register_default_execution_environments
 mkdir -p /awx_devel/awx/public/static
 mkdir -p /awx_devel/awx/ui/static
 mkdir -p /awx_devel/awx/ui_next/build/static
+
+# Create resource entries when using Minikube
+if [[ -n "$MINIKUBE_CONTAINER_GROUP" ]]; then
+    awx-manage shell_plus --quiet < /awx_devel/tools/docker-compose-minikube/_sources/bootstrap_minikube.py
+fi

tools/docker-compose/receptor.conf

@@ -1,12 +1,16 @@
 ---
-- log-level: info
+- node:
+    id: foo
+
+- log-level: debug
+
+- tcp-listener:
+    port: 2222
 
 - control-service:
     service: control
     filename: /var/run/receptor/receptor.sock
 
-- local-only:
-
 - work-command:
     worktype: local
     command: ansible-runner