Merge pull request #9156 from shanemcd/minikube-dev-env

Minikube-based development environment

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.gitignore

@@ -149,3 +149,6 @@ use_dev_supervisor.txt
 /tools/docker-compose/overrides/
 /awx/ui_next/.ui-built
 /Dockerfile
+/_build/
+/_build_kube_dev/
+/Dockerfile.kube-dev

Makefile

@@ -267,11 +267,27 @@ collectstatic:
 	fi; \
 	mkdir -p awx/public/static && $(PYTHON) manage.py collectstatic --clear --noinput > /dev/null 2>&1
 
+UWSGI_DEV_RELOAD_COMMAND ?= supervisorctl restart tower-processes:awx-dispatcher tower-processes:awx-receiver
+
 uwsgi: collectstatic
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-    uwsgi -b 32768 --socket 127.0.0.1:8050 --module=awx.wsgi:application --home=/var/lib/awx/venv/awx --chdir=/awx_devel/ --vacuum --processes=5 --harakiri=120 --master --no-orphans --py-autoreload 1 --max-requests=1000 --stats /tmp/stats.socket --lazy-apps --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" --hook-accepting1="exec:supervisorctl restart tower-processes:awx-dispatcher tower-processes:awx-receiver"
+	uwsgi -b 32768 \
+	    --socket 127.0.0.1:8050 \
+	    --module=awx.wsgi:application \
+	    --home=/var/lib/awx/venv/awx \
+	    --chdir=/awx_devel/ \
+	    --vacuum \
+	    --processes=5 \
+	    --harakiri=120 --master \
+	    --no-orphans \
+	    --py-autoreload 1 \
+	    --max-requests=1000 \
+	    --stats /tmp/stats.socket \
+	    --lazy-apps \
+	    --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" \
+	    --hook-accepting1="exec: $(UWSGI_DEV_RELOAD_COMMAND)"
 
 daphne:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -579,15 +595,18 @@ docker-compose-clean: awx/projects
 
 # Base development image build
 docker-compose-build:
-	ansible localhost -m template -a "src=installer/roles/image_build/templates/Dockerfile.j2 dest=tools/docker-compose/Dockerfile" -e build_dev=True
-	docker build -t ansible/awx_devel -f tools/docker-compose/Dockerfile \
-		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
+	ansible-playbook installer/dockerfile.yml -e build_dev=True
+	docker build -t ansible/awx_devel \
+	    --build-arg BUILDKIT_INLINE_CACHE=1 \
+	    --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
 	docker tag ansible/awx_devel $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
 	#docker push $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
 
 # For use when developing on "isolated" AWX deployments
 docker-compose-isolated-build: docker-compose-build
-	docker build -t ansible/awx_isolated -f tools/docker-isolated/Dockerfile .
+	docker build -t ansible/awx_isolated \
+	    --build-arg BUILDKIT_INLINE_CACHE=1 \
+	    -f tools/docker-isolated/Dockerfile .
 	docker tag ansible/awx_isolated $(DEV_DOCKER_TAG_BASE)/awx_isolated:$(COMPOSE_TAG)
 	#docker push $(DEV_DOCKER_TAG_BASE)/awx_isolated:$(COMPOSE_TAG)
 
@@ -624,5 +643,16 @@ psql-container:
 VERSION:
 	@echo "awx: $(VERSION)"
 
-Dockerfile: installer/roles/image_build/templates/Dockerfile.j2
-	ansible localhost -m template -a "src=installer/roles/image_build/templates/Dockerfile.j2 dest=Dockerfile"
+Dockerfile: installer/roles/dockerfile/templates/Dockerfile.j2
+	ansible-playbook installer/dockerfile.yml
+
+Dockerfile.kube-dev: installer/roles/dockerfile/templates/Dockerfile.j2
+	ansible-playbook installer/dockerfile.yml \
+	    -e dockerfile_name=Dockerfile.kube-dev \
+	    -e kube_dev=True \
+	    -e template_dest=_build_kube_dev
+
+awx-kube-dev-build: Dockerfile.kube-dev
+	docker build -f Dockerfile.kube-dev \
+	    --build-arg BUILDKIT_INLINE_CACHE=1 \
+	    -t $(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) .

awx/settings/development.py

@@ -158,7 +158,10 @@ AWX_VENV_PATH = os.path.join(BASE_VENV_PATH, "awx")
 # default settings for development.  If not present, we can still run using
 # only the defaults.
 try:
-    include(optional('local_*.py'), scope=locals())
+    if os.getenv('AWX_KUBE_DEVEL', False):
+        include(optional('minikube.py'), scope=locals())
+    else:
+        include(optional('local_*.py'), scope=locals())
 except ImportError:
     traceback.print_exc()
     sys.exit(1)

awx/settings/minikube.py

@@ -0,0 +1,4 @@
+BROADCAST_WEBSOCKET_SECRET = '🤖starscream🤖'
+BROADCAST_WEBSOCKET_PORT = 8013
+BROADCAST_WEBSOCKET_VERIFY_CERT = False
+BROADCAST_WEBSOCKET_PROTOCOL = 'http'

docs/development/minikube.md

@@ -0,0 +1,104 @@
+# Running Development Environment in Kubernetes
+
+## Start Minikube
+
+If you do not already have Minikube, install it from:
+https://minikube.sigs.k8s.io/docs/start/
+
+Note: This environment has only been tested on Linux.
+
+```
+$ minikube start \
+    --mount \
+    --mount-string="/path/to/awx:/awx_devel" \
+    --cpus=4 \
+    --memory=8g \
+    --addons=ingress
+```
+
+### Verify
+
+Ensure that your AWX source code is properly mounted inside of the minikube node:
+
+```
+$ minikube ssh
+$ ls -la /awx_devel
+```
+
+## Deploy the AWX Operator
+
+Clone the [awx-operator](https://github.com/ansible/awx-operator).
+
+For the following playbooks to work, you will need to:
+
+```
+$ pip install openshift
+```
+
+If you are not changing any code in the operator itself, simply run:
+
+```
+$ ansible-playbook ansible/deploy-operator.yml
+```
+
+If making changes to the operator itself, run the following command in the root
+of the awx-operator repo. If not, continue to the next section.
+
+### Building and Deploying a Custom AWX Operator Image
+
+```
+$ operator-sdk build quay.io/<username>/awx-operator
+$ docker push quay.io/<username>/awx-operator
+$ ansible-playbook ansible/deploy-operator.yml \
+    -e pull_policy=Always \
+    -e operator_image=quay.io/<username>/awx-operator \
+    -e operator_version=latest
+```
+
+## Deploy AWX into Minikube using the AWX Operator
+
+If have have not made any changes to the AWX Dockerfile, run the following
+command. If you need to test out changes to the Dockerfile, see the 
+"Custom AWX Development Image for Kubernetes" section below.
+
+In the root of awx-operator:
+
+```
+$ ansible-playbook ansible/instantiate-awx-deployment.yml \
+    -e development_mode=yes \
+    -e tower_image=gcr.io/ansible-tower-engineering/awx_kube_devel:devel \
+    -e tower_image_pull_policy=Always \
+    -e tower_ingress_type=ingress
+```
+
+### Custom AWX Development Image for Kubernetes
+
+I have found `minikube cache add` to be unacceptably slow for larger images such
+as this. A faster workflow involves building the image and pushing it to a
+registry:
+
+In the root of the AWX repo:
+
+```
+$ make awx-kube-dev-build
+$ docker push gcr.io/ansible-tower-engineering/awx_kube_devel:${COMPOSE_TAG}
+```
+
+In the root of awx-operator:
+
+```
+$ ansible-playbook ansible/instantiate-awx-deployment.yml \
+    -e development_mode=yes \
+    -e tower_image=gcr.io/ansible-tower-engineering/awx_kube_devel:${COMPOSE_TAG} \
+    -e tower_image_pull_policy=Always \
+    -e tower_ingress_type=ingress
+```
+
+To iterate on changes to the Dockerfile, rebuild and push the image, then delete
+the AWX Pod. A new Pod will respawn with the latest revision. 
+
+## Accessing AWX
+
+```
+$ minikube service awx-service --url
+```

installer/build.yml

@@ -3,5 +3,6 @@
   hosts: localhost
   gather_facts: true
   roles:
+    - {role: dockerfile}
     - {role: image_build}
     - {role: image_push, when: "docker_registry is defined"}

installer/dockerfile.yml

@@ -0,0 +1,6 @@
+---
+- name: Render AWX Dockerfile and sources
+  hosts: localhost
+  gather_facts: true
+  roles:
+    - {role: dockerfile}

installer/roles/dockerfile/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+build_dev: false
+kube_dev: false
+dockerfile_dest: '..'
+dockerfile_name: 'Dockerfile'
+template_dest: '_build'

installer/roles/dockerfile/files/launch_awx.sh

@@ -5,6 +5,14 @@ if [ `id -u` -ge 500 ]; then
     rm /tmp/passwd
 fi
 
+if [ -n "${AWX_KUBE_DEVEL}" ]; then
+    pushd /awx_devel
+    make awx-link
+    popd
+
+    export SDB_NOTIFY_HOST=$(ip route | head -n1 | awk '{print $3}')
+fi
+
 source /etc/tower/conf.d/environment.sh
 
 ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all

installer/roles/dockerfile/files/launch_awx_task.sh

@@ -5,6 +5,14 @@ if [ `id -u` -ge 500 ]; then
     rm /tmp/passwd
 fi
 
+if [ -n "${AWX_KUBE_DEVEL}" ]; then
+    pushd /awx_devel
+    make awx-link
+    popd
+
+    export SDB_NOTIFY_HOST=$(ip route | head -n1 | awk '{print $3}')
+fi
+
 source /etc/tower/conf.d/environment.sh
 
 ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all

installer/roles/dockerfile/tasks/main.yml

@@ -0,0 +1,19 @@
+---
+
+- name: Create .build directory
+  file:
+    path: "{{ dockerfile_dest }}/{{ template_dest }}"
+    state: directory
+
+- name: Render supervisor configs
+  template:
+    src: "{{ item }}.j2"
+    dest: "{{ dockerfile_dest }}/{{ template_dest }}/{{ item }}"
+  with_items:
+    - "supervisor.conf"
+    - "supervisor_task.conf"
+
+- name: Render Dockerfile
+  template:
+    src: Dockerfile.j2
+    dest: "{{ dockerfile_dest }}/{{ dockerfile_name }}"

installer/roles/dockerfile/templates/Dockerfile.j2

@@ -1,12 +1,8 @@
-{% if build_dev|default(False)|bool %}
 ### This file is generated from
-### installer/roles/image_build/templates/Dockerfile.j2
+### installer/roles/dockerfile/templates/Dockerfile.j2
 ###
 ### DO NOT EDIT
 ###
-{% else %}
-     {% set build_dev = False  %}
-{% endif %}
 
 # Locations - set globally to be used across stages
 ARG COLLECTION_BASE="/var/lib/awx/vendor/awx_ansible_collections"
@@ -67,12 +63,10 @@ ADD requirements/requirements_ansible.txt \
 RUN cd /tmp && make requirements_awx requirements_ansible_py3
 RUN cd /tmp && make requirements_collections
 
-{% if build_dev|bool %}
+{% if (build_dev|bool) or (kube_dev|bool) %}
 ADD requirements/requirements_dev.txt /tmp/requirements
 RUN cd /tmp && make requirements_awx_dev requirements_ansible_dev
-{% endif %}
-
-{% if not build_dev|bool %}
+{% else %}
 # Use the distro provided npm to bootstrap our required version of node
 RUN npm install -g n && n 14.15.1 && dnf remove -y nodejs
 
@@ -81,6 +75,7 @@ COPY . /tmp/src/
 WORKDIR /tmp/src/
 RUN make sdist && \
     /var/lib/awx/venv/awx/bin/pip install dist/awx-$(cat VERSION).tar.gz
+RUN ln -s /var/lib/awx/venv/awx/bin/awx-manage /usr/bin/awx-manage
 {% endif %}
 
 # Final container(s)
@@ -146,7 +141,7 @@ RUN cd /usr/local/bin && \
     curl -L https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz | \
     tar -xz --strip-components=1 --wildcards --no-anchored 'oc'
 
-{% if build_dev|bool %}
+{% if (build_dev|bool) or (kube_dev|bool) %}
 # Install development/test requirements
 RUN dnf -y install \
     gdb \
@@ -183,32 +178,32 @@ RUN openssl req -nodes -newkey rsa:2048 -keyout /etc/nginx/nginx.key -out /etc/n
         -subj "/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost" && \
     openssl x509 -req -days 365 -in /etc/nginx/nginx.csr -signkey /etc/nginx/nginx.key -out /etc/nginx/nginx.crt && \
     chmod 640 /etc/nginx/nginx.{csr,key,crt}
-{% else %}
-RUN ln -s /var/lib/awx/venv/awx/bin/awx-manage /usr/bin/awx-manage
 {% endif %}
 
 # Create default awx rsyslog config
-ADD installer/roles/image_build/files/rsyslog.conf /var/lib/awx/rsyslog/rsyslog.conf
+ADD installer/roles/dockerfile/files/rsyslog.conf /var/lib/awx/rsyslog/rsyslog.conf
 
 ## File mappings
 {% if build_dev|bool %}
 ADD tools/docker-compose/launch_awx.sh /usr/bin/launch_awx.sh
-ADD tools/docker-compose/awx-manage /usr/local/bin/awx-manage
-ADD tools/docker-compose/awx.egg-link /tmp/awx.egg-link
 ADD tools/docker-compose/nginx.conf /etc/nginx/nginx.conf
 ADD tools/docker-compose/nginx.vh.default.conf /etc/nginx/conf.d/nginx.vh.default.conf
 ADD tools/docker-compose/start_tests.sh /start_tests.sh
 ADD tools/docker-compose/bootstrap_development.sh /usr/bin/bootstrap_development.sh
 ADD tools/docker-compose/entrypoint.sh /entrypoint.sh
-ADD tools/scripts/awx-python /usr/bin/awx-python
 {% else %}
-ADD installer/roles/image_build/files/launch_awx.sh /usr/bin/launch_awx.sh
-ADD installer/roles/image_build/files/launch_awx_task.sh /usr/bin/launch_awx_task.sh
-ADD installer/roles/image_build/files/settings.py /etc/tower/settings.py
-ADD installer/roles/image_build/files/supervisor.conf /etc/supervisord.conf
-ADD installer/roles/image_build/files/supervisor_task.conf /etc/supervisord_task.conf
+ADD installer/roles/dockerfile/files/launch_awx.sh /usr/bin/launch_awx.sh
+ADD installer/roles/dockerfile/files/launch_awx_task.sh /usr/bin/launch_awx_task.sh
+ADD installer/roles/dockerfile/files/settings.py /etc/tower/settings.py
+ADD {{ template_dest }}/supervisor.conf /etc/supervisord.conf
+ADD {{ template_dest }}/supervisor_task.conf /etc/supervisord_task.conf
 ADD tools/scripts/config-watcher /usr/bin/config-watcher
 {% endif %}
+{% if (build_dev|bool) or (kube_dev|bool) %}
+ADD tools/docker-compose/awx.egg-link /tmp/awx.egg-link
+ADD tools/docker-compose/awx-manage /usr/local/bin/awx-manage
+ADD tools/scripts/awx-python /usr/bin/awx-python
+{% endif %}
 
 # Pre-create things we need to access
 RUN for dir in \
@@ -231,7 +226,7 @@ RUN chmod u+s /usr/bin/bwrap ; \
     chgrp -R root ${COLLECTION_BASE} ; \
     chmod -R g+rw ${COLLECTION_BASE}
 
-{% if build_dev|bool %}
+{% if (build_dev|bool) or (kube_dev|bool) %}
 RUN for dir in \
       /var/lib/awx/venv \
       /var/lib/awx/venv/awx/lib/python3.6 \
@@ -256,6 +251,7 @@ ENV HOME="/var/lib/awx"
 ENV PATH="/usr/pgsql-10/bin:${PATH}"
 
 {% if build_dev|bool %}
+
 EXPOSE 8043 8013 8080 22
 
 ENTRYPOINT ["/entrypoint.sh"]

installer/roles/dockerfile/templates/supervisor.conf.j2

@@ -6,7 +6,12 @@ logfile_maxbytes = 0
 pidfile = /var/run/supervisor/supervisor.web.pid
 
 [program:nginx]
+{% if kube_dev | bool %}
+command = make nginx
+directory = /awx_devel
+{% else %}
 command = nginx -g "daemon off;"
+{% endif %}
 autostart = true
 autorestart = true
 stopwaitsecs = 5
@@ -16,34 +21,59 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
 [program:uwsgi]
+
+{% if kube_dev | bool %}
+command = make uwsgi
+directory = /awx_devel
+environment =
+    UWSGI_DEV_RELOAD_COMMAND='supervisorctl -c /etc/supervisord_task.conf restart all; supervisorctl restart tower-processes:daphne tower-processes:wsbroadcast'
+{% else %}
 command = /var/lib/awx/venv/awx/bin/uwsgi --socket 127.0.0.1:8050 --module=awx.wsgi:application --vacuum --processes=5 --harakiri=120 --no-orphans --master --max-requests=1000 --master-fifo=/var/lib/awx/awxfifo --lazy-apps -b 32768
 directory = /var/lib/awx
+{% endif %}
 autostart = true
 autorestart = true
 stopwaitsecs = 15
-stopsignal = INT
+stopasgroup=true
+killasgroup=true
+stopsignal=KILL
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
 [program:daphne]
+{% if kube_dev | bool %}
+command = make daphne
+directory = /awx_devel
+{% else %}
 command = /var/lib/awx/venv/awx/bin/daphne -b 127.0.0.1 -p 8051 --websocket_timeout -1 awx.asgi:channel_layer
 directory = /var/lib/awx
+{% endif %}
 autostart = true
+stopsignal=KILL
 autorestart = true
 stopwaitsecs = 5
+stopasgroup=true
+killasgroup=true
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
 [program:wsbroadcast]
+{% if kube_dev | bool %}
+command = make wsbroadcast
+directory = /awx_devel
+{% else %}
 command = awx-manage run_wsbroadcast
 directory = /var/lib/awx
+{% endif %}
 autostart = true
 autorestart = true
 stopwaitsecs = 5
+stopasgroup=true
+killasgroup=true
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
@@ -53,6 +83,7 @@ stderr_logfile_maxbytes=0
 command = rsyslogd -n -i /var/run/awx-rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
 autostart = true
 autorestart = true
+startretries = 10
 stopwaitsecs = 5
 stopsignal=TERM
 stopasgroup=true

installer/roles/dockerfile/templates/supervisor_task.conf.j2

@@ -6,8 +6,13 @@ logfile_maxbytes = 0
 pidfile = /var/run/supervisor/supervisor.pid
 
 [program:dispatcher]
+{% if kube_dev | bool %}
+command = make dispatcher
+directory = /awx_devel
+{% else %}
 command = awx-manage run_dispatcher
 directory = /var/lib/awx
+{% endif %}
 autostart = true
 autorestart = true
 stopwaitsecs = 5
@@ -17,8 +22,13 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
 [program:callback-receiver]
+{% if kube_dev | bool %}
+command = make receiver
+directory = /awx_devel
+{% else %}
 command = awx-manage run_callback_receiver
 directory = /var/lib/awx
+{% endif %}
 autostart = true
 autorestart = true
 stopwaitsecs = 5

installer/roles/image_build/defaults/main.yml

@@ -1,6 +1,5 @@
 ---
 create_preload_data: true
-build_dev: false
 
 # Helper vars to construct the proper download URL for the current architecture
 tini_architecture: '{{ { "x86_64": "amd64", "aarch64": "arm64", "armv7": "arm" }[ansible_facts.architecture] }}'

installer/roles/image_build/tasks/main.yml

@@ -21,11 +21,6 @@
   set_fact:
     awx_image: "{{ awx_image|default('awx') }}"
 
-- name: Render Dockerfile
-  template:
-    src: Dockerfile.j2
-    dest: ../Dockerfile
-
 # Calling Docker directly because docker-py doesnt support BuildKit
 - name: Build AWX image
   command: docker build -t {{ awx_image }}:{{ awx_version }} ..

installer/roles/kubernetes/templates/supervisor.yml.j2

@@ -61,6 +61,7 @@ data:
     autostart = true
     autorestart = true
     stopwaitsecs = 5
+    startretries = 10
     stopsignal=TERM
     stopasgroup=true
     killasgroup=true