From ec390b049dd9ff38885f23c53f5d475ea2c4ac82 Mon Sep 17 00:00:00 2001 From: Vismay Golwala Date: Tue, 26 Feb 2019 22:09:28 -0500 Subject: [PATCH] Feature: custom virtual environment directories Currently, users are allowed to define virtual environments in `settings.BASE_VENV_PATH` only, because that's the only place Tower looks for virtual environments. This feature allows users to custom define the directory paths, using API or UI, to look for virtual environments. Tower aggregates virtual environments from all these paths, except environments with special name `awx`. Signed-off-by: Vismay Golwala --- awx/conf/fields.py | 21 ++++++ awx/conf/tests/unit/test_fields.py | 48 +++++++++++++- awx/main/conf.py | 11 ++++ awx/main/tests/unit/utils/test_common.py | 13 +++- awx/main/utils/common.py | 27 ++++---- awx/settings/defaults.py | 2 + .../system-form/sub-forms/system-misc.form.js | 4 ++ docs/custom_virtualenvs.md | 65 +++++++++++++------ 8 files changed, 156 insertions(+), 35 deletions(-) diff --git a/awx/conf/fields.py b/awx/conf/fields.py index 52a63e0d7d..e48ef80506 100644 --- a/awx/conf/fields.py +++ b/awx/conf/fields.py @@ -1,4 +1,5 @@ # Python +import os import logging import urllib.parse as urlparse from collections import OrderedDict @@ -96,6 +97,26 @@ class StringListBooleanField(ListField): self.fail('type_error', input_type=type(data)) +class StringListPathField(StringListField): + + default_error_messages = { + 'type_error': _('Expected list of strings but got {input_type} instead.'), + 'path_error': _('{path} is not a valid path choice.'), + } + + def to_internal_value(self, paths): + if isinstance(paths, (list, tuple)): + for p in paths: + if not isinstance(p, str): + self.fail('type_error', input_type=type(p)) + if not os.path.exists(p): + self.fail('path_error', path=p) + + return super(StringListPathField, self).to_internal_value(sorted({os.path.normpath(path) for path in paths})) + else: + self.fail('type_error', input_type=type(paths)) + + class URLField(CharField): def __init__(self, **kwargs): diff --git a/awx/conf/tests/unit/test_fields.py b/awx/conf/tests/unit/test_fields.py index 0126723e97..e1e01a86e9 100644 --- a/awx/conf/tests/unit/test_fields.py +++ b/awx/conf/tests/unit/test_fields.py @@ -1,7 +1,7 @@ import pytest from rest_framework.fields import ValidationError -from awx.conf.fields import StringListBooleanField, ListTuplesField +from awx.conf.fields import StringListBooleanField, StringListPathField, ListTuplesField class TestStringListBooleanField(): @@ -84,3 +84,49 @@ class TestListTuplesField(): assert e.value.detail[0] == "Expected a list of tuples of max length 2 " \ "but got {} instead.".format(t) + +class TestStringListPathField(): + + FIELD_VALUES = [ + ((".", "..", "/"), [".", "..", "/"]), + (("/home",), ["/home"]), + (("///home///",), ["/home"]), + (("/home/././././",), ["/home"]), + (("/home", "/home", "/home/"), ["/home"]), + (["/home/", "/home/", "/opt/", "/opt/", "/var/"], ["/home", "/opt", "/var"]) + ] + + FIELD_VALUES_INVALID_TYPE = [ + 1.245, + {"a": "b"}, + ("/home"), + ] + + FIELD_VALUES_INVALID_PATH = [ + "", + "~/", + "home", + "/invalid_path", + "/home/invalid_path", + ] + + @pytest.mark.parametrize("value_in, value_known", FIELD_VALUES) + def test_to_internal_value_valid(self, value_in, value_known): + field = StringListPathField() + v = field.to_internal_value(value_in) + assert v == value_known + + @pytest.mark.parametrize("value", FIELD_VALUES_INVALID_TYPE) + def test_to_internal_value_invalid_type(self, value): + field = StringListPathField() + with pytest.raises(ValidationError) as e: + field.to_internal_value(value) + assert e.value.detail[0] == "Expected list of strings but got {} instead.".format(type(value)) + + @pytest.mark.parametrize("value", FIELD_VALUES_INVALID_PATH) + def test_to_internal_value_invalid_path(self, value): + field = StringListPathField() + with pytest.raises(ValidationError) as e: + field.to_internal_value([value]) + assert e.value.detail[0] == "{} is not a valid path choice.".format(value) + diff --git a/awx/main/conf.py b/awx/main/conf.py index 5472c930d2..06ab279823 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -126,6 +126,17 @@ register( category_slug='system', ) +register( + 'CUSTOM_VENV_PATHS', + field_class=fields.StringListPathField, + label=_('Custom virtual environment paths'), + help_text=_('Paths where Tower will look for custom virtual environments ' + '(in addition to /var/lib/awx/venv/). Enter one path per line.'), + category=_('System'), + category_slug='system', + default=[], +) + register( 'AD_HOC_COMMANDS', field_class=fields.StringListField, diff --git a/awx/main/tests/unit/utils/test_common.py b/awx/main/tests/unit/utils/test_common.py index f7e9f68bbc..6568b1b486 100644 --- a/awx/main/tests/unit/utils/test_common.py +++ b/awx/main/tests/unit/utils/test_common.py @@ -177,9 +177,18 @@ def test_get_custom_venv_choices(): with TemporaryDirectory(dir=settings.BASE_VENV_PATH, prefix='tmp') as temp_dir: os.makedirs(os.path.join(temp_dir, 'bin', 'activate')) - assert sorted(common.get_custom_venv_choices()) == [ + + custom_venv_dir = os.path.join(temp_dir, 'custom') + custom_venv_1 = os.path.join(custom_venv_dir, 'venv-1') + custom_venv_awx = os.path.join(custom_venv_dir, 'custom', 'awx') + + os.makedirs(os.path.join(custom_venv_1, 'bin', 'activate')) + os.makedirs(os.path.join(custom_venv_awx, 'bin', 'activate')) + + assert sorted(common.get_custom_venv_choices([custom_venv_dir])) == [ bundled_venv, - os.path.join(temp_dir, '') + os.path.join(temp_dir, ''), + os.path.join(custom_venv_1, '') ] diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 34f58a0b41..3539f11536 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -943,19 +943,22 @@ def get_current_apps(): return current_apps -def get_custom_venv_choices(): +def get_custom_venv_choices(custom_paths=None): from django.conf import settings - custom_venv_path = settings.BASE_VENV_PATH - if os.path.exists(custom_venv_path): - return [ - os.path.join(custom_venv_path, x, '') - for x in os.listdir(custom_venv_path) - if x != 'awx' and - os.path.isdir(os.path.join(custom_venv_path, x)) and - os.path.exists(os.path.join(custom_venv_path, x, 'bin', 'activate')) - ] - else: - return [] + custom_paths = custom_paths or settings.CUSTOM_VENV_PATHS + all_venv_paths = [settings.BASE_VENV_PATH] + custom_paths + custom_venv_choices = [] + + for custom_venv_path in all_venv_paths: + if os.path.exists(custom_venv_path): + custom_venv_choices.extend([ + os.path.join(custom_venv_path, x, '') + for x in os.listdir(custom_venv_path) + if x != 'awx' and + os.path.isdir(os.path.join(custom_venv_path, x)) and + os.path.exists(os.path.join(custom_venv_path, x, 'bin', 'activate')) + ]) + return custom_venv_choices class OutputEventFilter(object): diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 709eee8e66..9fdf03163f 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -166,6 +166,8 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST'] # REMOTE_HOST_HEADERS will be trusted unconditionally') PROXY_IP_WHITELIST = [] +CUSTOM_VENV_PATHS = [] + # Note: This setting may be overridden by database settings. STDOUT_MAX_BYTES_DISPLAY = 1048576 diff --git a/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-misc.form.js b/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-misc.form.js index f9dddd237b..e15713ee39 100644 --- a/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-misc.form.js +++ b/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-misc.form.js @@ -54,6 +54,10 @@ export default ['i18n', function(i18n) { REMOTE_HOST_HEADERS: { type: 'textarea', reset: 'REMOTE_HOST_HEADERS' + }, + CUSTOM_VENV_PATHS: { + type: 'textarea', + reset: 'CUSTOM_VENV_PATHS' } }, diff --git a/docs/custom_virtualenvs.md b/docs/custom_virtualenvs.md index 65657e43a6..32c946c826 100644 --- a/docs/custom_virtualenvs.md +++ b/docs/custom_virtualenvs.md @@ -16,29 +16,48 @@ virtualenv; this documentation describes the supported way to do so. Preparing a New Custom Virtualenv ================================= awx allows a _different_ virtualenv to be specified and used on Job Template -runs. To choose a custom virtualenv, first create one in `/var/lib/awx/venv`: +runs. To choose a custom virtualenv, first we need to create one. Here, we are +using `/opt/my-envs/` as the directory to hold custom venvs. But you can use any +other directory and replace `/opt/my-envs/` with that. Let's create the directory +first if absent: - $ sudo virtualenv /var/lib/awx/venv/my-custom-venv + $ sudo mkdir /opt/my-envs + +Now, we need to tell Tower to look into this directory for custom venvs. For that, +we can add this directory to the `CUSTOM_VENV_PATHS` setting as: + + $ HTTP PATCH /api/v2/settings/system {'CUSTOM_VENV_PATHS': ["/opt/my-envs/"]} + +If we have venvs spanned over multiple directories, we can add all the paths and +Tower will aggregate venvs from them: + + $ HTTP PATCH /api/v2/settings/system {'CUSTOM_VENV_PATHS': ["/path/1/to/venv/", + "/path/2/to/venv/", + "/path/3/to/venv/"]} + +Now that we have the directory setup, we can create a virtual environment in that using: + + $ sudo virtualenv /opt/my-envs/custom-venv Multiple versions of Python are supported, though it's important to note that the semantics for creating virtualenvs in Python 3 has changed slightly: - $ sudo python3 -m venv /var/lib/awx/venv/my-custom-venv + $ sudo python3 -m venv /opt/my-envs/custom-venv Your newly created virtualenv needs a few base dependencies to properly run playbooks: fact gathering): - $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install psutil + $ sudo /opt/my-envs/custom-venv/bin/pip install psutil From here, you can install _additional_ Python dependencies that you care about, such as a per-virtualenv version of ansible itself: - $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install -U "ansible == X.Y.Z" + $ sudo /opt/my-envs/custom-venv/bin/pip install -U "ansible == X.Y.Z" ...or an additional third-party SDK that's not included with the base awx installation: - $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install -U python-digitalocean + $ sudo /opt/my-envs/custom-venv/bin/pip install -U python-digitalocean If you want to copy them, the libraries included in awx's default virtualenv can be found using `pip freeze`: @@ -47,7 +66,7 @@ can be found using `pip freeze`: One important item to keep in mind is that in a clustered awx installation, you need to ensure that the same custom virtualenv exists on _every_ local file -system at `/var/lib/awx/venv/`. For container-based deployments, this likely +system at `/opt/my-envs/`. For container-based deployments, this likely means building these steps into your own custom image building workflow, e.g., ```diff @@ -60,8 +79,9 @@ index aa8b304..eb05f91 100644 fi +requirements_custom: -+ virtualenv $(VENV_BASE)/my-custom-env -+ $(VENV_BASE)/my-custom-env/bin/pip install psutil ++ mkdir -p /opt/my-envs ++ virtualenv /opt/my-envs/my-custom-env ++ /opt/my-envs/my-custom-env/bin/pip install psutil + diff --git a/installer/image_build/templates/Dockerfile.j2 b/installer/image_build/templates/Dockerfile.j2 index d69e2c9..a08bae5 100644 @@ -74,6 +94,8 @@ index d69e2c9..a08bae5 100644 + VENV_BASE=/var/lib/awx/venv make requirements_custom && \ ``` +Once the AWX API is available, update the `CUSTOM_VENV_PATHS` setting as described in `Preparing a New Custom Virtualenv`. + Kubernetes Custom Virtualenvs ============================= @@ -87,7 +109,7 @@ Now create an initContainer stanza. You can subsititute your own custom images initContainers: - image: 'centos:7' - name: init-my-custom-venv + name: init-custom-venv command: - sh - '-c' @@ -95,19 +117,20 @@ Now create an initContainer stanza. You can subsititute your own custom images yum install -y ansible python-pip curl python-setuptools epel-release openssl openssl-devel gcc python-devel && curl 'https://bootstrap.pypa.io/get-pip.py' | python && pip install virtualenv && - virtualenv /var/lib/awx/venv/my-custom-venv && - source /var/lib/awx/venv/my-custom-venv/bin/activate && - /var/lib/awx/venv/my-custom-venv/bin/pip install psutil && - /var/lib/awx/venv/my-custom-venv/bin/pip install -U "ansible == X.Y.Z" && - /var/lib/awx/venv/my-custom-venv/bin/pip install -U custom-python-module + mkdir -p /opt/my-envs && + virtualenv /opt/my-envs/custom-venv && + source /opt/my-envs/custom-venv/bin/activate && + /opt/my-envs/custom-venv/bin/pip install psutil && + /opt/my-envs/custom-venv/bin/pip install -U "ansible == X.Y.Z" && + /opt/my-envs/custom-venv/bin/pip install -U custom-python-module volumeMounts: - - mountPath: /var/lib/awx/venv/my-custom-venv + - mountPath: /opt/my-envs/custom-venv name: custom-venv Finally in the awx-celery and awx-web containers stanza add the shared volume as a mount. volumeMounts: - - mountPath: /var/lib/awx/venv/my-custom-venv + - mountPath: /opt/my-envs/custom-venv name: custom-venv - mountPath: /etc/tower name: awx-application-config @@ -116,6 +139,8 @@ Finally in the awx-celery and awx-web containers stanza add the shared volume as name: awx-confd readOnly: true +Once the AWX API is available, update the `CUSTOM_VENV_PATHS` setting as described in `Preparing a New Custom Virtualenv`. + Assigning Custom Virtualenvs ============================ Once you've created a custom virtualenv, you can assign it at the Organization, @@ -127,7 +152,7 @@ Project, or Job Template level: Content-Type: application/json { - 'custom_virtualenv': '/var/lib/awx/venv/my-custom-venv' + 'custom_virtualenv': '/opt/my-envs/custom-venv' } An HTTP `GET` request to `/api/v2/config/` will provide a list of @@ -135,8 +160,8 @@ detected installed virtualenvs: { "custom_virtualenvs": [ - "/var/lib/awx/venv/my-custom-venv", - "/var/lib/awx/venv/my-other-custom-venv", + "/opt/my-envs/custom-venv", + "/opt/my-envs/my-other-custom-venv", ], ... }