Editable dependencies in docker compose development environment (#14979)

* Editable dependencies in docker compose development environment

tools/docker-compose/ansible/roles/sources/defaults/main.yml

@@ -59,3 +59,7 @@ scrape_interval: '5s'
 enable_pgbouncer: false
 pgbouncer_port: 6432
 pgbouncer_max_pool_size: 70
+
+# editable_dependencies
+install_editable_dependencies: false
+editable_dependencies: []

tools/docker-compose/ansible/roles/sources/tasks/main.yml

@@ -105,6 +105,30 @@
   include_tasks: vault_tls.yml
   when: enable_vault | bool
 
+- name: Iterate through ../editable_dependencies and get symlinked directories and register the paths
+  find:
+    paths: "{{ playbook_dir }}/../editable_dependencies"
+    file_type: link
+    recurse: no
+  register: _editable_dependencies_links
+  when: install_editable_dependencies | bool
+
+- name: Warn about empty editable_dependnecies
+  fail:
+    msg: "[WARNING] No editable_dependencies found in ../editable_dependencies"
+  when: install_editable_dependencies | bool and not _editable_dependencies_links.files
+  ignore_errors: true
+
+- name: Set fact with editable_dependencies
+  set_fact:
+    editable_dependencies: "{{ _editable_dependencies_links.files | map(attribute='path') | list }}"
+  when: install_editable_dependencies | bool and _editable_dependencies_links.files
+
+- name: Set install_editable_dependnecies to false if no editable_dependencies are found
+  set_fact:
+    install_editable_dependencies: false
+  when: install_editable_dependencies | bool and not _editable_dependencies_links.files
+
 - name: Render Docker-Compose
   template:
     src: docker-compose.yml.j2

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -2,6 +2,22 @@
 ---
 version: '2.1'
 services:
+{% if editable_dependencies | length > 0 %}
+  init_awx:
+    image: "{{ awx_image }}:{{ awx_image_tag }}"
+    container_name: tools_init_awx
+    command: /awx_devel/tools/docker-compose/editable_dependencies/install.sh
+    user: root
+    working_dir: "/"
+    environment:
+      RUN_MIGRATIONS: 1
+    volumes:
+      - "../../../:/awx_devel"
+{% for editable_dependency in editable_dependencies %}
+      - "{{ editable_dependency }}:/editable_dependencies/{{ editable_dependency | basename }}"
+{% endfor %}
+      - "var_lib_awx:/var/lib/awx"
+{% endif %}
 {% for i in range(control_plane_node_count|int) %}
   {% set container_postfix = loop.index %}
   {% set awx_sdb_port_start = 7899 + (loop.index0*1000) | int %}
@@ -39,6 +55,12 @@ services:
       - service-mesh
     working_dir: "/awx_devel"
     volumes:
+{% if editable_dependencies | length > 0 %}
+      - "var_lib_awx:/var/lib/awx"
+{% for editable_dependency in editable_dependencies %}
+      - "{{ editable_dependency }}:/editable_dependencies/{{ editable_dependency | basename }}"
+{% endfor %}
+{% endif %}
       - "../../../:/awx_devel"
       - "../../docker-compose/supervisor.conf:/etc/supervisord.conf"
       - "../../docker-compose/_sources/database.py:/etc/tower/conf.d/database.py"
@@ -58,6 +80,11 @@ services:
       - "~/.kube/config:/var/lib/awx/.kube/config"
       - "redis_socket_{{ container_postfix }}:/var/run/redis/:rw"
     privileged: true
+{% if editable_dependencies | length > 0 %}
+    depends_on:
+      init_awx:
+        condition: service_completed_successfully
+{% endif %}
     tty: true
     ports:
       - "{{ awx_sdb_port_start }}-{{ awx_sdb_port_end }}:{{ awx_sdb_port_start }}-{{ awx_sdb_port_end }}"  # sdb-listen
@@ -304,6 +331,10 @@ services:
 {% endif %}
 
 volumes:
+{% if editable_dependencies | length > 0 %}
+  var_lib_awx:
+    name: tools_var_lib_awx
+{% endif %}
 {# For the postgres 15 db upgrade we changed the mount name because 15 can't load a 12 DB #}
   awx_db_15:
     name: tools_awx_db_15
@@ -329,6 +360,7 @@ volumes:
   grafana_storage:
     name: tools_grafana_storage
 {% endif %}
+
 networks:
   awx:
   service-mesh:

tools/docker-compose/editable_dependencies/README.md

@@ -0,0 +1,66 @@
+# Editable dependencies in AWX Docker Compose Development Environment
+
+This folder contains the symlink to editable dependencies for AWX
+
+During the bootstrap of awx development environment we will try to crawl through the symlinks and mount (the source of the symlink) to `tools_awx_` containers and `init_awx` containers than install all the dependencies in editable mode
+
+## How to enable/disable editable dependnecies
+
+### Enable
+
+Set `EDITABLE_DEPENDENCIES=true` either as an Environment Variable with before invoking `make docker-compose`
+
+```bash
+export EDITABLE_DEPENDENCIES=true
+```
+
+or during invocation of `make docker-compose`
+
+```bash
+
+EDITABLE_DEPENDENCIES=true make docker-compose
+```
+
+will cause the `make docker-compose-source` to template out docker-compose file with editable dependencies.
+
+### Disable
+
+To disable editable dependency simply `unset EDITABLE_DEPENDENCIES`
+
+## How to add editable dependencies
+
+Adding symlink to the directory that contains the source of the editable dependencies will cause the dependency to be mounted and installed in the docker-compose development environment.
+
+Both relative path or absolute path will work
+
+### Examples
+
+I have `awx` checked out at `~/projects/src/github.com/TheRealHaoLiu/awx`
+I have `django-ansible-base` checked out at `~/projects/src/github.com/TheRealHaoLiu/ansible-runner`
+
+From root of AWX project `~/projects/src/github.com/TheRealHaoLiu/awx`
+
+I can either do
+
+```bash
+ln -s ~/projects/src/github.com/TheRealHaoLiu/ansible-runner tools/docker-compose/editable_dependencies/
+```
+
+or
+
+```bash
+ln -s ../ansible-runner tools/docker-compose/editable_dependencies/
+```
+
+## How to remove indivisual editable dependencies
+
+Simply removing the symlink from  `tools/docker-compose/editable_dependencies` **will cause problem**!
+
+and the volume `tools_awx_var_lib` need to be deleted with
+
+```bash
+make docker-compose-down
+docker volume rm tools_awx_var_lib
+```
+
+TODO(TheRealHaoLiu): bear proof this? maybe just always delete tools_awx_var_lib?

tools/docker-compose/editable_dependencies/install.sh

@@ -0,0 +1,4 @@
+for file in `ls /editable_dependencies`; do
+    echo "Installing $file"
+    pip install -e /editable_dependencies/$file
+done