diff --git a/.gitignore b/.gitignore index 26e0f20f0a..6176ad6aed 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,11 @@ tools/docker-compose/overrides/ tools/docker-compose-minikube/_sources tools/docker-compose/keycloak.awx.realm.json +!tools/docker-compose/editable_dependencies +tools/docker-compose/editable_dependencies/* +!tools/docker-compose/editable_dependencies/README.md +!tools/docker-compose/editable_dependencies/install.sh + # Tower setup playbook testing setup/test/roles/postgresql **/provision_docker diff --git a/Makefile b/Makefile index 11c98dea70..5926e8674c 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,8 @@ VAULT ?= false VAULT_TLS ?= false # If set to true docker-compose will also start a tacacs+ instance TACACS ?= false +# If set to true docker-compose will install editable dependencies +EDITABLE_DEPENDENCIES ?= false VENV_BASE ?= /var/lib/awx/venv @@ -533,6 +535,7 @@ docker-compose-sources: .git/hooks/pre-commit -e enable_vault=$(VAULT) \ -e vault_tls=$(VAULT_TLS) \ -e enable_tacacs=$(TACACS) \ + -e install_editable_dependencies=$(EDITABLE_DEPENDENCIES) \ $(EXTRA_SOURCES_ANSIBLE_OPTS) docker-compose: awx/projects docker-compose-sources @@ -540,9 +543,15 @@ docker-compose: awx/projects docker-compose-sources ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/initialize_containers.yml \ -e enable_vault=$(VAULT) \ -e vault_tls=$(VAULT_TLS) \ - -e enable_ldap=$(LDAP); + -e enable_ldap=$(LDAP); \ + $(MAKE) docker-compose-up + +docker-compose-up: $(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans +docker-compose-down: + $(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) down --remove-orphans + docker-compose-credential-plugins: awx/projects docker-compose-sources echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m" $(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx_1 --remove-orphans @@ -607,7 +616,7 @@ docker-clean: -$(foreach image_id,$(shell docker images --filter=reference='*/*/*awx_devel*' --filter=reference='*/*awx_devel*' --filter=reference='*awx_devel*' -aq),docker rmi --force $(image_id);) docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean - docker volume rm -f tools_awx_db tools_vault_1 tools_ldap_1 tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q) + docker volume rm -f tools_var_lib_awx tools_awx_db tools_vault_1 tools_ldap_1 tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q) docker-refresh: docker-clean docker-compose diff --git a/tools/docker-compose/ansible/roles/sources/defaults/main.yml b/tools/docker-compose/ansible/roles/sources/defaults/main.yml index 9362be4ee8..647aabd5a1 100644 --- a/tools/docker-compose/ansible/roles/sources/defaults/main.yml +++ b/tools/docker-compose/ansible/roles/sources/defaults/main.yml @@ -59,3 +59,7 @@ scrape_interval: '5s' enable_pgbouncer: false pgbouncer_port: 6432 pgbouncer_max_pool_size: 70 + +# editable_dependencies +install_editable_dependencies: false +editable_dependencies: [] diff --git a/tools/docker-compose/ansible/roles/sources/tasks/main.yml b/tools/docker-compose/ansible/roles/sources/tasks/main.yml index 4c30528865..0f1149053e 100644 --- a/tools/docker-compose/ansible/roles/sources/tasks/main.yml +++ b/tools/docker-compose/ansible/roles/sources/tasks/main.yml @@ -105,6 +105,30 @@ include_tasks: vault_tls.yml when: enable_vault | bool +- name: Iterate through ../editable_dependencies and get symlinked directories and register the paths + find: + paths: "{{ playbook_dir }}/../editable_dependencies" + file_type: link + recurse: no + register: _editable_dependencies_links + when: install_editable_dependencies | bool + +- name: Warn about empty editable_dependnecies + fail: + msg: "[WARNING] No editable_dependencies found in ../editable_dependencies" + when: install_editable_dependencies | bool and not _editable_dependencies_links.files + ignore_errors: true + +- name: Set fact with editable_dependencies + set_fact: + editable_dependencies: "{{ _editable_dependencies_links.files | map(attribute='path') | list }}" + when: install_editable_dependencies | bool and _editable_dependencies_links.files + +- name: Set install_editable_dependnecies to false if no editable_dependencies are found + set_fact: + install_editable_dependencies: false + when: install_editable_dependencies | bool and not _editable_dependencies_links.files + - name: Render Docker-Compose template: src: docker-compose.yml.j2 diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 71cb57e2ad..461d065b78 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -2,6 +2,22 @@ --- version: '2.1' services: +{% if editable_dependencies | length > 0 %} + init_awx: + image: "{{ awx_image }}:{{ awx_image_tag }}" + container_name: tools_init_awx + command: /awx_devel/tools/docker-compose/editable_dependencies/install.sh + user: root + working_dir: "/" + environment: + RUN_MIGRATIONS: 1 + volumes: + - "../../../:/awx_devel" +{% for editable_dependency in editable_dependencies %} + - "{{ editable_dependency }}:/editable_dependencies/{{ editable_dependency | basename }}" +{% endfor %} + - "var_lib_awx:/var/lib/awx" +{% endif %} {% for i in range(control_plane_node_count|int) %} {% set container_postfix = loop.index %} {% set awx_sdb_port_start = 7899 + (loop.index0*1000) | int %} @@ -39,6 +55,12 @@ services: - service-mesh working_dir: "/awx_devel" volumes: +{% if editable_dependencies | length > 0 %} + - "var_lib_awx:/var/lib/awx" +{% for editable_dependency in editable_dependencies %} + - "{{ editable_dependency }}:/editable_dependencies/{{ editable_dependency | basename }}" +{% endfor %} +{% endif %} - "../../../:/awx_devel" - "../../docker-compose/supervisor.conf:/etc/supervisord.conf" - "../../docker-compose/_sources/database.py:/etc/tower/conf.d/database.py" @@ -58,6 +80,11 @@ services: - "~/.kube/config:/var/lib/awx/.kube/config" - "redis_socket_{{ container_postfix }}:/var/run/redis/:rw" privileged: true +{% if editable_dependencies | length > 0 %} + depends_on: + init_awx: + condition: service_completed_successfully +{% endif %} tty: true ports: - "{{ awx_sdb_port_start }}-{{ awx_sdb_port_end }}:{{ awx_sdb_port_start }}-{{ awx_sdb_port_end }}" # sdb-listen @@ -304,6 +331,10 @@ services: {% endif %} volumes: +{% if editable_dependencies | length > 0 %} + var_lib_awx: + name: tools_var_lib_awx +{% endif %} {# For the postgres 15 db upgrade we changed the mount name because 15 can't load a 12 DB #} awx_db_15: name: tools_awx_db_15 @@ -329,6 +360,7 @@ volumes: grafana_storage: name: tools_grafana_storage {% endif %} + networks: awx: service-mesh: diff --git a/tools/docker-compose/editable_dependencies/README.md b/tools/docker-compose/editable_dependencies/README.md new file mode 100644 index 0000000000..aa67cebb50 --- /dev/null +++ b/tools/docker-compose/editable_dependencies/README.md @@ -0,0 +1,66 @@ +# Editable dependencies in AWX Docker Compose Development Environment + +This folder contains the symlink to editable dependencies for AWX + +During the bootstrap of awx development environment we will try to crawl through the symlinks and mount (the source of the symlink) to `tools_awx_` containers and `init_awx` containers than install all the dependencies in editable mode + +## How to enable/disable editable dependnecies + +### Enable + +Set `EDITABLE_DEPENDENCIES=true` either as an Environment Variable with before invoking `make docker-compose` + +```bash +export EDITABLE_DEPENDENCIES=true +``` + +or during invocation of `make docker-compose` + +```bash + +EDITABLE_DEPENDENCIES=true make docker-compose +``` + +will cause the `make docker-compose-source` to template out docker-compose file with editable dependencies. + +### Disable + +To disable editable dependency simply `unset EDITABLE_DEPENDENCIES` + +## How to add editable dependencies + +Adding symlink to the directory that contains the source of the editable dependencies will cause the dependency to be mounted and installed in the docker-compose development environment. + +Both relative path or absolute path will work + +### Examples + +I have `awx` checked out at `~/projects/src/github.com/TheRealHaoLiu/awx` +I have `django-ansible-base` checked out at `~/projects/src/github.com/TheRealHaoLiu/ansible-runner` + +From root of AWX project `~/projects/src/github.com/TheRealHaoLiu/awx` + +I can either do + +```bash +ln -s ~/projects/src/github.com/TheRealHaoLiu/ansible-runner tools/docker-compose/editable_dependencies/ +``` + +or + +```bash +ln -s ../ansible-runner tools/docker-compose/editable_dependencies/ +``` + +## How to remove indivisual editable dependencies + +Simply removing the symlink from `tools/docker-compose/editable_dependencies` **will cause problem**! + +and the volume `tools_awx_var_lib` need to be deleted with + +```bash +make docker-compose-down +docker volume rm tools_awx_var_lib +``` + +TODO(TheRealHaoLiu): bear proof this? maybe just always delete tools_awx_var_lib? diff --git a/tools/docker-compose/editable_dependencies/install.sh b/tools/docker-compose/editable_dependencies/install.sh new file mode 100755 index 0000000000..55bd940e1b --- /dev/null +++ b/tools/docker-compose/editable_dependencies/install.sh @@ -0,0 +1,4 @@ +for file in `ls /editable_dependencies`; do + echo "Installing $file" + pip install -e /editable_dependencies/$file +done